gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974  Split orchestrator_functions into pipeline configs, tag database and OPC slots
- orchestrator_functions.py held four unrelated concerns (pipeline
  builders, tag database indexes, PI Web API resolution, OPC slot v2)
  and had grown to 1135 lines. Move them, unchanged, into:
  pipeline_configs.py, tag_database.py and opc_slots.py.
- Make the helpers shared across modules public: is_complete,
  tag_id_text and pi_connection.
- Split the utils tests the same way, with the pipeline and tag
  database documents they share in tag_database_fixtures.py.
- Point formatters, the orchestrator workflow, README and CLAUDE.md at
  the new modules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-28 16:38:53 -03:00
parent 11e350a6ce
commit 9eaec54fc8
14 changed files with 2777 additions and 2706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
orchestrator/activities/formatters.py
View File

@@ -17,24 +17,25 @@ with workflow.unsafe.imports_passed_through():
now,
)
from orchestrator.utils.orchestrator_functions import (
PI_OUTPUT_WORKFLOW_TYPES,
build_tag_config,
from orchestrator.utils.opc_slots import build_tag_config, gather_read_tags
from orchestrator.utils.pipeline_configs import (
chunk_period_warning,
drift,
frequency_alert_warning,
gather_read_tags,
index_connector_tags,
index_connectors,
index_pi_connectors,
minimal_retrain,
pi_web_api_scouter,
predictions_batch,
scouter,
simple_metrics,
)
from orchestrator.utils.tag_database import (
PI_OUTPUT_WORKFLOW_TYPES,
index_connector_tags,
index_connectors,
index_pi_connectors,
resolve_pi_output_tags,
resolve_pi_read_tags,
resolve_tag_names,
scouter,
simple_metrics,
)
topic_separator = '\n ========== \n'

282
orchestrator/utils/opc_slots.py Normal file
View File

@@ -0,0 +1,282 @@
"""OPC slot v2 (contract C1): read tags grouped by server and address for the gateway."""
from typing import Any, cast
from orchestrator.utils.tag_database import is_complete, resolve_tag_names, tag_id_text
def _connector_server(connector: dict[str, Any]) -> dict[str, Any]:
"""Slot server fields of a tag database connector (connectors collection)."""
return {
'server_id': str(connector['connector_id']),
'name': connector['name'],
'url': connector['url'],
'server_uri': connector['server_uri'],
'use_security': bool(connector.get('use_security', False)),
'subscription_period_ms': connector.get('subscription_period_ms'),
'from_tag_database': True,
}
def _opc_server(server_id: str, opc_server: dict[str, Any]) -> dict[str, Any]:
"""
Slot server fields of a legacy opc-servers document.
The certificate now always comes from MinIO on the gateway side, so the file paths
are not carried: a cert_path only tells that the server uses security (RF4). The
subscription period is left to build_tag_config (half the lowest tag frequency).
"""
return {
'server_id': server_id,
'name': opc_server['server_name'],
'url': opc_server['url'],
'server_uri': opc_server['uri'],
'use_security': bool(opc_server.get('cert_path')),
'subscription_period_ms': None,
'from_tag_database': False,
}
def _frequency(tag: dict[str, Any]) -> int | None:
"""The read tag's frequency as int, or None when it is missing or not an integer."""
try:
return int(tag['frequency'])
except (KeyError, TypeError, ValueError):
return None
def _locate_read_tag(
tag: dict[str, Any],
opc_servers: dict[str, dict[str, Any]],
connectors: dict[str, dict[str, Any]],
tags_by_id: dict[str, dict[str, Any]],
issues: dict[str, set[str]],
) -> tuple[dict[str, Any], str] | None:
"""
Server and address of a read tag, or None when they cannot be resolved.
A tag with tag_id takes its address and connector from connector_tags and its server
from connectors (RF3). A tag without one keeps the current fields: server_id +
tag_address, with the server from opc-servers (RF4). What cannot be resolved is
recorded in issues ('tags', 'connectors' or 'servers').
"""
tag_id = tag_id_text(tag)
if tag_id is not None:
connector_tag = tags_by_id.get(tag_id)
if connector_tag is None:
issues['tags'].add(tag_id)
return None
connector_id = str(connector_tag['connector_id'])
connector = connectors.get(connector_id)
if connector is None:
issues['connectors'].add(connector_id)
return None
return _connector_server(connector), connector_tag['address']
server_id = str(tag['server_id'])
opc_server = opc_servers.get(server_id)
if opc_server is None:
issues['servers'].add(server_id)
return None
return _opc_server(server_id, opc_server), tag['tag_address']
def _invalid_read_tag(tag: dict[str, Any]) -> bool:
"""
A read tag the slot cannot use: no integer frequency, or (without tag_id) no
server_id, tag_address or tag_name.
"""
if _frequency(tag) is None:
return True
if tag_id_text(tag) is not None:
return False
return not is_complete(tag, ('server_id', 'tag_address', 'tag_name'))
def _merge_active_read(current: dict[str, Any], requested: dict[str, Any] | None) -> dict[str, Any]:
"""Active read of an address: on if any pipeline asks, with the lowest cycle_multiple (D3)."""
if not requested or not requested.get('enabled'):
return current
cycle_multiple = int(requested.get('cycle_multiple', 1))
if current.get('enabled'):
cycle_multiple = min(cycle_multiple, current['cycle_multiple'])
return {'enabled': True, 'cycle_multiple': cycle_multiple}
def _assign_server_keys(servers: dict[str, dict[str, Any]]) -> None:
"""
Give each server a unique slot key: its name, or name:server_id (extended until
free) when another server already took it. Assigned in server_id order, so the keys
are the same on every run and in every slot.
"""
taken: set[str] = set()
for server_id in sorted(servers):
server = servers[server_id]
key = server['name']
while key in taken:
key = f'{key}:{server_id}'
server['key'] = key
taken.add(key)
def _add_read_tag(
entries: dict[tuple[str, str], dict[str, Any]],
servers: dict[str, dict[str, Any]],
pipeline: dict[str, Any],
tag: dict[str, Any],
server: dict[str, Any],
address: str,
) -> None:
"""Merge one resolved read tag into its (server, address) entry and its server."""
server_id = server['server_id']
known = servers.get(server_id)
if known is None or (server['from_tag_database'] and not known['from_tag_database']):
servers[server_id] = server
# int, so the slot does not depend on how each pipeline stored the number
frequency = cast(int, _frequency(tag))
entry = entries.setdefault(
(server_id, address),
{
'address': address,
'frequency': frequency,
'active_read': {'enabled': False},
'names': [],
},
)
entry['frequency'] = min(entry['frequency'], frequency)
entry['active_read'] = _merge_active_read(entry['active_read'], tag.get('active_read'))
name = {
'collection': f'raw_{pipeline["schedule_name"]}',
'tag_name': tag['tag_name'],
'model_id': pipeline['model_id'],
'model_name': pipeline['model']['name'],
}
if name not in entry['names']:
entry['names'].append(name)
def gather_read_tags(
pipelines: list[dict[str, Any]],
opc_servers: dict[str, dict[str, Any]],
connectors: dict[str, dict[str, Any]],
tags_by_id: dict[str, dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, set[str]]]:
"""
Group the read tags of all scouter pipelines by (server, address) for the slot v2.
An address read by N pipelines becomes ONE entry with N items in names[]: the
gateway reads it once and writes it to N collections (SIENTIAPDE-1974, RF2). Each
read tag is resolved through the tag database when it has a tag_id (RF3), or through
the current server_id + tag_address + opc-servers fields when it has not (RF4).
A server known through both connectors and opc-servers under the same server_id is
the same server (the gateway keys its connections by server_id); the connectors
fields win.
Args:
pipelines (list[dict[str, Any]]): Pipeline configurations. Only workflow_type
'scouter' is read; each needs schedule_name, model_id, model.name and
read_tags (with frequency, and optionally tag_id and active_read)
opc_servers (dict[str, dict[str, Any]]): opc-servers documents keyed by str(id)
connectors (dict[str, dict[str, Any]]): Active OPC UA connectors indexed by
index_connectors
tags_by_id (dict[str, dict[str, Any]]): Active Read connector_tags indexed by
index_connector_tags
Returns:
tuple[list[dict[str, Any]], dict[str, set[str]]]:
- One entry per (server, address), sorted by server_id then address, with:
- server (dict): server_id, name, url, server_uri, use_security,
subscription_period_ms (None = derive from the frequencies) and key
(the server's unique slot key)
- address (str): The OPC node address
- frequency (int): The lowest frequency among the pipelines (RF6)
- active_read (dict): {'enabled': False} unless a pipeline enables it (D3)
- names (list[dict]): collection ('raw_{schedule_name}'), tag_name,
model_id and model_name, one per pipeline name, sorted
- What was left out: 'tags' (tag_id), 'connectors' (connector_id), 'servers'
(opc-servers server_id) and 'invalid' (schedule_name/read tag without a
usable frequency or legacy fields)
"""
issues: dict[str, set[str]] = {
'tags': set(),
'connectors': set(),
'servers': set(),
'invalid': set(),
}
servers: dict[str, dict[str, Any]] = {}
entries: dict[tuple[str, str], dict[str, Any]] = {}
for pipeline in pipelines:
if pipeline['workflow_type'] != 'scouter':
continue
for tag in resolve_tag_names(pipeline.get('read_tags') or [], tags_by_id):
if _invalid_read_tag(tag):
ref = tag_id_text(tag) or tag.get('tag_address') or tag.get('tag_name')
issues['invalid'].add(f'{pipeline["schedule_name"]}/{ref}')
continue
located = _locate_read_tag(tag, opc_servers, connectors, tags_by_id, issues)
if located is None:
continue
_add_read_tag(entries, servers, pipeline, tag, *located)
_assign_server_keys(servers)
gathered = []
for (server_id, _address), entry in sorted(entries.items()):
# A stable order keeps the slot identical between runs, so the gateway does
# not re-subscribe an unchanged slot (O6)
entry['names'].sort(key=lambda n: (n['collection'], n['tag_name'], str(n['model_id'])))
gathered.append({'server': servers[server_id], **entry})
return gathered, issues
def build_tag_config(entries: list[dict[str, Any]]) -> dict[str, Any]:
"""
Build one slot in the slot v2 format (contract C1) from address entries.
The slot no longer carries cert_path, private_key_path or server_cert_path: the
gateway takes the certificate from MinIO (SIENTIAPDE-1974, O4).
Args:
entries (list[dict[str, Any]]): Address entries from gather_read_tags for this slot
Returns:
dict[str, Any]: The slot keyed by server key, each server with server_id, name,
url, server_uri, use_security, subscription_period_ms and tags. Each tag (keyed
by address) has frequency, active_read and names. A server without a
subscription period of its own (opc-servers) gets half its lowest tag
frequency in this slot, as before.
"""
slot_config: dict[str, Any] = {}
for entry in entries:
server = entry['server']
if server['key'] not in slot_config:
slot_config[server['key']] = {
'server_id': server['server_id'],
'name': server['name'],
'url': server['url'],
'server_uri': server['server_uri'],
'use_security': server['use_security'],
'subscription_period_ms': server['subscription_period_ms'],
'tags': {},
}
slot_config[server['key']]['tags'][entry['address']] = {
'frequency': entry['frequency'],
'active_read': entry['active_read'],
'names': entry['names'],
}
for server_config in slot_config.values():
if server_config['subscription_period_ms'] is None:
frequencies = [tag['frequency'] for tag in server_config['tags'].values()]
server_config['subscription_period_ms'] = min(frequencies) / 2
return slot_config

1135
orchestrator/utils/orchestrator_functions.py
View File

File diff suppressed because it is too large Load Diff

513
orchestrator/utils/pipeline_configs.py Normal file
View File

@@ -0,0 +1,513 @@
"""Pipeline configuration builders: one Temporal schedule input per pipeline type."""
from math import ceil
from typing import Any
from orchestrator.utils.converters import parse_frequency
from orchestrator.utils.tag_database import PI_QUERY_DEFAULTS, pi_connection
def common_config(config: dict[str, Any]):
"""
Extract common configuration parameters from a pipeline configuration.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- workflow_type (str): Type of workflow (e.g., 'scouter', 'predictions_batch', 'drift')
- schedule_name (str): Unique name identifier for this schedule
- model_id (str): MongoDB ID of the associated model
- model (dict): Model configuration containing:
- name (str): Human-readable name of the model
- model_config (dict, optional): Additional model-specific configuration
- frequency (str, optional): Execution frequency (default: '1m')
Format: '{number}{unit}' where unit is 's', 'm', 'h', or 'd'
- offset (str, optional): Schedule offset/delay (default: '0m')
- max_retry_policy (int, optional): Maximum retry attempts on failure (default: 1)
- execution_timeout_seconds (int, optional): Workflow execution timeout (default: 300)
- task_timeout_seconds (int, optional): Individual task timeout (default: 300)
Returns:
dict[str, Any]: Common configuration dictionary with standardized parameters
for Temporal workflow execution
"""
model = config['model']
return {
'workflow_type': config['workflow_type'],
'schedule_name': config['schedule_name'],
'frequency': config.get('frequency', '1m'),
'offset': config.get('offset', '0m'),
'max_retry_policy': config.get('max_retry_policy', 1),
'model_id': config['model_id'],
'model_name': model['name'],
'model_config': model.get('model_config', {}),
'execution_timeout_seconds': config.get('execution_timeout_seconds', 300),
'task_timeout_seconds': config.get('task_timeout_seconds', 300),
'on_conflict': config.get('on_conflict', 'error'),
'runtime': config.get('runtime', 'legacy'),
}
# Kept in sync by hand with the `chunk_period` validation in
# sientia-dataops-laborious/laborious/activities/model_metrics.py (no shared contract).
DRIFT_CHUNK_PERIODS = ('min', 's')
def _drift_interval_minutes(config: dict[str, Any], frequency: Any) -> int:
"""
Drift window in minutes: `interval_minutes` as configured, otherwise the schedule frequency
rounded up to whole minutes (min 1). Consecutive default windows do not overlap, so each
drift chunk is stored once.
"""
if 'interval_minutes' in config:
return config['interval_minutes']
try:
seconds = parse_frequency(str(frequency))
except ValueError:
# create_schedules fails this schedule on its own; don't kill the tick for every model.
return 60
return max(1, ceil(seconds / 60))
def drift(config: dict[str, Any]):
"""
Build drift configuration from pipeline config.
Creates a drift detection workflow configuration with database table mappings
and drift metric specifications for monitoring data distribution changes.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- interval_minutes (int, optional): Time window for data comparison in minutes,
used as-is. Default: `frequency` rounded up to whole minutes (min 1), so
consecutive windows do not overlap (1 with the default '1m');
`interval_minutes: 60` keeps the legacy window.
- chunk_period (str, optional): Drift bucket size, 'min' or 's' (default: 'min').
Any other value falls back to 'min' and create_schedule_config warns.
- drift_metrics (list[str], optional): Statistical metrics to compute (default:
['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein'])
- Additional fields from common_config
Returns:
dict[str, Any]: Drift detection configuration with source/target tables,
time interval, chunk period and metrics specifications. Results stored in
'drift_metrics' table
"""
base = common_config(config)
configured_chunk_period = config.get('chunk_period')
chunk_period_valid = configured_chunk_period in DRIFT_CHUNK_PERIODS
output = {
**base,
'schema': 'sientia_data',
'source_table_name': 'laborious_data',
'target_table_name': 'drift_metrics',
'interval': _drift_interval_minutes(config, base['frequency']),
'chunk_period': configured_chunk_period if chunk_period_valid else 'min',
'drift_metrics': config.get(
'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
),
}
if configured_chunk_period is not None and not chunk_period_valid:
# Forwarded so create_schedule_config can warn without re-reading the pipeline.
output['chunk_period_configured'] = configured_chunk_period
return output
def chunk_period_warning(chunk_period: Any) -> str | None:
"""
Warning message when a configured drift `chunk_period` is not supported by laborious,
otherwise None. Warn only: `drift()` already replaced it with 'min'.
"""
if chunk_period is None or chunk_period in DRIFT_CHUNK_PERIODS:
return None
return (
f"chunk_period={chunk_period!r} is not one of {DRIFT_CHUNK_PERIODS}; falling back to 'min'"
)
def simple_metrics(config: dict[str, Any]):
"""
Build simple metrics configuration from pipeline config.
Creates a simple metrics computation workflow configuration for calculating
model performance metrics like RMSE, MSE, MAE, and R².
Args:
config (dict[str, Any]): Pipeline configuration containing:
- interval_minutes (int, optional): Computation interval in minutes (default: 60)
- metrics (list[str], optional): List of metrics to compute
(default: ['rmse', 'mse', 'mae', 'r2'])
- frequency_alert (str, optional): Frequency used while `adaptive_state.escalated`
is true; same format as `frequency`. Absent = adaptive frequency off.
- adaptive_state (dict, optional): `{escalated, healthy_streak}`, written by laborious.
- Additional fields from common_config
Returns:
dict[str, Any]: Simple metrics configuration with source tables (predictions and
actual data), target table, time interval, and metrics list. Results stored in
'simple_metrics' table
"""
base = common_config(config)
frequency_alert = config.get('frequency_alert')
# SIENTIAPDE-1995: laborious flips `adaptive_state.escalated` on a threshold breach (and back
# after the window is renewed without one) and bumps `updated_at`, which makes this tick
# rewrite the schedule.
state = config.get('adaptive_state')
# Hand-written docs: a malformed state must not raise and kill the tick for every model.
if frequency_alert and isinstance(state, dict) and state.get('escalated') is True:
try:
# Temporal parses the schedule frequency on create/update; an unparseable value
# would fail it, so keep `frequency` (create_schedule_config still warns).
parse_frequency(str(frequency_alert))
base['frequency'] = frequency_alert
except ValueError:
pass
return {
**base,
'schema': 'sientia_data',
'predictions_table_name': 'predictions',
'data_table_name': 'laborious_data',
'target_table_name': 'simple_metrics',
'interval_minutes': config.get('interval_minutes', 60),
'metrics': config.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
# Forwarded so create_schedule_config can warn on a value below the floor.
'frequency_alert': frequency_alert,
}
MIN_FREQUENCY_ALERT_SECONDS = 60
def frequency_alert_warning(frequency_alert: str | None) -> str | None:
"""
Warning message when `frequency_alert` is below the 1-minute floor or does not parse,
otherwise None. Warn only (SIENTIAPDE-1995 decision 7): a value below the floor is kept;
an unparseable one is ignored by `simple_metrics()`, which keeps `frequency`.
"""
if not frequency_alert:
return None
try:
# str(): a non-string value (e.g. a number typed into the doc) must warn, not raise
# AttributeError and fail the whole create_schedule_config activity.
seconds = parse_frequency(str(frequency_alert))
except ValueError:
return f'frequency_alert={frequency_alert!r} is not a valid frequency'
if seconds < MIN_FREQUENCY_ALERT_SECONDS:
return f'frequency_alert={frequency_alert} is below the 1m floor; kept as configured'
return None
def minimal_retrain(config: dict[str, Any]):
"""
Build minimal retrain configuration from pipeline config.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- query (str): SQL query to retrieve training data. Should return features
and target variable in expected format
- datetime_columns (list[str], optional): Column names to parse as datetime
for proper temporal handling (default: [])
- Additional fields from common_config
Returns:
dict[str, Any]: Minimal retrain configuration with SQL query, database settings,
and datetime column specifications. Retraining logs are stored in 'log_retrain' table
"""
return {
**common_config(config),
'query': config['query'],
'schema': 'sientia_data',
'table_name': 'log_retrain',
'datetime_columns': config.get('datetime_columns', []),
}
def base_scouter(config: dict[str, Any]):
"""
Build base scouter configuration shared by all scouter workflow types.
Creates the foundational configuration for OPC data collection workflows,
including filter policies, database settings, and data retention parameters.
This configuration is extended by specific scouter implementations (OPC UA, PI Web API).
Args:
config (dict[str, Any]): Pipeline configuration containing:
- filters (list[dict], optional): List of filter configurations with:
- filter_name (str): Name of the filter
- policy (str): Filter policy to apply
- tag_retention_minutes (int, optional): Tag retention time in minutes (default: 60)
- debug_data_package (bool, optional): Enable debug data package logging (default: False)
- fill_missing_tags (bool, optional): Fill missing tags with interpolation (default: False)
- Additional fields from common_config
Returns:
dict[str, Any]: Base scouter configuration with filters, database settings,
and retention policies
"""
filters = {}
for f in config.get('filters', []):
filters[f['filter_name']] = {'policy': f['policy']}
return {
**common_config(config),
'trigger_laborious': False,
'filters': filters,
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': config.get('tag_retention_minutes', 60) * 60,
'debug_data_package': config.get('debug_data_package', False),
'fill_missing_tags': config.get('fill_missing_tags', False),
}
def scouter(config: dict[str, Any]):
"""
Build scouter configuration from pipeline config.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- schedule_name (str): Name of the schedule (used for topic generation)
- read_tags (list[dict]): List of tag configurations, already passed through
resolve_tag_names (process_schedules does it), with:
- tag_name (str): Name of the tag to read. A tag without one (a tag_id the
tag database cannot resolve) is left out; process_slots notifies it
- aggr_func (str, optional): Aggregation function for data collection (default: 'lts')
Common values: 'lts' (last), 'avg' (average), 'min', 'max', 'sum'
- data_range (list[int], optional): Valid data range [min, max] (default: [-100, 100])
- Additional fields from base_scouter
Returns:
dict[str, Any]: OPC UA scouter configuration with Kafka topic, tag mappings,
filters, and retention settings. Topic name follows pattern: 'raw_{schedule_name}'
"""
tags = {}
for tag in config['read_tags']:
if not tag.get('tag_name'):
continue
tags[tag['tag_name']] = {
'aggr_func': tag.get('aggr_func', 'lts'),
'data_range': tag.get('data_range', [-100, 100]),
}
return {
**base_scouter(config),
'topic': f'raw_{config["schedule_name"]}',
'model_tags': tags,
}
def pi_web_api_scouter(config: dict[str, Any]):
"""
Build PI Web API scouter configuration from pipeline config.
Creates a data collection workflow configuration for OSIsoft PI servers using
the PI Web API REST interface. Configures tag mappings with WebIDs, aggregation
functions, and API query parameters including timeout management.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- read_tags (dict[str, dict]): Tag configurations keyed by model tag name, already
passed through resolve_pi_read_tags (process_schedules does it), with:
- webid (str): PI Web API WebID for the tag
- aggr_func (str, optional): Aggregation function (default: 'lts')
- data_range (list[int], optional): Valid data range (default: [-100, 100])
- pi_web_api_config (dict): PI Web API query settings with:
- endpoint (str): PI Web API endpoint URL
- period (str, optional): Time period for data retrieval (default: '*-1d')
- max_count (int, optional): Maximum number of values to retrieve (default: 1)
- api_timeout (int, optional): API request timeout in seconds
- pi_web_api_connector (dict, optional): The tag database connector of the
pipeline's resolved tags (set by process_schedules). Its pi_web_api period,
max_count and api_timeout are defaults under pi_web_api_config
- Additional fields from base_scouter
Returns:
dict[str, Any]: PI Web API scouter configuration with tag mappings and query settings.
API timeout is automatically adjusted to not exceed workflow frequency. With a
connector, it also carries connection: {connector_id, base_url, auth} (auth holds
the secret reference only, never the credential).
"""
tags = {}
for tag, tag_config in config['read_tags'].items():
tags[tag] = {
'webid': tag_config['webid'],
'aggr_func': tag_config.get('aggr_func', 'lts'),
'data_range': tag_config.get('data_range', [-100, 100]),
}
base_config = base_scouter(config)
connector = config.get('pi_web_api_connector')
pi_web_api_config = config['pi_web_api_config']
if connector:
connector_defaults = connector['pi_web_api']
pi_web_api_config = {
**{
key: connector_defaults[key]
for key in PI_QUERY_DEFAULTS
if key in connector_defaults
},
**pi_web_api_config,
}
config_timeout = pi_web_api_config.get('api_timeout', None)
frequency = parse_frequency(base_config['frequency'])
if config_timeout is None or config_timeout > frequency:
config_timeout = frequency
output = {
**base_config,
'model_tags': tags,
'pi_web_api_query': {
'endpoint': pi_web_api_config['endpoint'],
'period': pi_web_api_config.get('period', '*-1d'),
'max_count': pi_web_api_config.get('max_count', 1),
'api_timeout': config_timeout,
},
}
if connector:
output['connection'] = pi_connection(connector)
return output
def overlap_filter_config(base_filter_config: dict[str, Any], config: list[dict[str, Any]]):
"""
Merge filter configurations with base filter configuration.
Extends the base filter configuration dictionary by adding or overwriting
filters from the provided configuration list. Used in predictions_batch
workflows to combine default filters with user-defined custom filters.
Args:
base_filter_config (dict[str, Any]): Base filter configuration dictionary to extend.
Each filter entry contains 'policy' and optionally 'config' keys.
config (list[dict[str, Any]]): List of filter configurations to merge, each containing:
- filter_name (str): Name of the filter to add or update
- policy (str): Filter policy (e.g., 'STOP', 'CONTINUE', 'REPEAT')
- config (dict, optional): Additional filter-specific configuration
Returns:
dict[str, Any]: Extended filter configuration dictionary with merged filters.
Filters from config list overwrite or add to base_filter_config entries.
"""
for fil in config:
base_filter_config[fil['filter_name']] = {
'policy': fil['policy'],
'config': fil.get('config', {}),
}
return base_filter_config
def process_path_priority(path_priority: list[str]):
"""
Process and normalize path priority list for filter policy execution order.
Validates and normalizes the priority list used to determine the order in which
filter policies are evaluated in prediction workflows. Invalid priorities are
removed, missing required priorities are appended, and the list is truncated to
exactly 3 elements.
Valid priorities define workflow behavior when filters are triggered:
- STOP: Halt workflow execution immediately
- CONTINUE: Proceed to next step despite filter trigger
- REPEAT: Retry the current step
Args:
path_priority (list[str]): User-provided list of path priorities. May contain
invalid values or be incomplete.
Returns:
list[str]: Normalized path priority list with exactly 3 elements in user-specified
or default order. Default order when priorities are missing: ["STOP", "CONTINUE", "REPEAT"]
"""
for priority in path_priority[:]:
if priority not in ['STOP', 'CONTINUE', 'REPEAT']:
path_priority.remove(priority)
for priority in ['STOP', 'CONTINUE', 'REPEAT']:
if priority not in path_priority:
path_priority.append(priority)
return path_priority[0:3]
def predictions_batch(config: dict[str, Any]):
"""
Build predictions batch configuration from pipeline config.
Args:
config (dict[str, Any]): Pipeline configuration containing:
- query (str): SQL query to retrieve input data for predictions
- write_tags (list[dict]): List of OPC tag configurations for write-back with:
- server_id (str): ID of the target OPC server
- type (str): Tag type - 'prediction' (model output) or 'confidence' (prediction confidence)
- addr (str): OPC tag address/path
- data_type (str, optional): OPC data type (default: 'float')
- datetime_columns (list[str], optional): Column names to parse as datetime (default: [])
- path_priority (list[str], optional): Filter policy execution order (default: ["STOP", "CONTINUE", "REPEAT"])
- input_filters (list[dict], optional): Input data validation filters
- mlflow_transform_filters (list[dict], optional): Transform stage filters
- mlflow_predict_filters (list[dict], optional): Prediction stage filters
- model_retention_minutes (int, optional): Data retention time in minutes (default: 60)
- save_transform (bool, optional): Save transformed data to database (default: True)
- predictions_storage_policy (str, optional): Prediction storage policy (default: 'lts:1')
- pi_web_api_output_config (dict, optional): PI Web API output configuration for write-back (default: {}),
already passed through resolve_pi_output_tags (process_schedules does it)
- Additional fields from common_config
Returns:
dict[str, Any]: Complete predictions batch configuration with OPC output mappings,
PI Web API output configuration, multi-stage filters, SQL query, and retention policies
"""
tags: dict[str, Any] = {}
for tag in config.get('write_tags', []):
if tag['server_id'] not in tags:
tags[tag['server_id']] = {}
tag_type = tag['type']
if tag_type == 'prediction' or tag_type == 'confidence':
tag_type_str = f'{tag_type}_tags'
if tag_type_str not in tags[tag['server_id']]:
tags[tag['server_id']][tag_type_str] = {}
tags[tag['server_id']][tag_type_str][tag['addr']] = {
'data_type': tag.get('data_type', 'float'),
}
path_priority = process_path_priority(
config.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT'])
)
return {
**common_config(config),
'query': config['query'],
'datetime_columns': config.get('datetime_columns', []),
'schema': 'sientia_data',
'table_name': 'predictions',
'save_transform': config.get('save_transform', True),
'transform_table_name': 'transformed_data',
'retention_time': config.get('model_retention_minutes', 60) * 60,
'opc_output_config': tags,
'pi_web_api_output_config': config.get('pi_web_api_output_config', {}),
'input_filters': overlap_filter_config(
{'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, config.get('input_filters', [])
),
'mlflow_transform_filters': overlap_filter_config(
{
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
config.get('mlflow_transform_filters', []),
),
'mlflow_predict_filters': overlap_filter_config(
{'API_ERROR': {'policy': 'STOP', 'config': {}}},
config.get('mlflow_predict_filters', []),
),
'path_priority': path_priority,
'predictions_storage_policy': config.get('predictions_storage_policy', 'lts:1'),
}

359
orchestrator/utils/tag_database.py Normal file
View File

@@ -0,0 +1,359 @@
"""
Tag database (connectors + connector_tags): which tags the pipelines reference, the
indexes of what was loaded, and the resolution of OPC UA names and PI Web API tags.
"""
from typing import Any, cast
# Fields a tag database document needs to take part in a slot. Documents missing any
# of them are left out of the index, so their read tags are notified as not found
# instead of failing the whole run.
CONNECTOR_TAG_FIELDS = ('tag_id', 'connector_id', 'address', 'tag_name')
CONNECTOR_FIELDS = ('connector_id', 'name', 'url', 'server_uri')
def is_complete(document: dict[str, Any], fields: tuple[str, ...]) -> bool:
return all(document.get(field) not in (None, '') for field in fields)
def tag_id_text(tag: dict[str, Any]) -> str | None:
"""The read tag's tag_id as text, or None when it has none (missing, null or empty)."""
tag_id = tag.get('tag_id')
return None if tag_id in (None, '') else str(tag_id)
def referenced_tag_ids(pipelines: list[dict[str, Any]]) -> list[Any]:
"""
The tag_ids the scouter pipelines reference, for the connector_tags query.
Each id goes in both forms, int and text, because the tag database stores int and
pipelines may hold either. Sorted, so the query is the same on every run.
Args:
pipelines (list[dict[str, Any]]): Pipeline configurations
Returns:
list[Any]: The referenced tag_ids (empty when no read tag carries a tag_id)
"""
tags = [
tag
for pipeline in pipelines
if pipeline.get('workflow_type') == 'scouter'
for tag in pipeline.get('read_tags') or []
]
return _query_tag_ids(tags)
def _query_tag_ids(tags: list[Any]) -> list[Any]:
"""The tag_ids of the tags in both forms (int and text), sorted for a stable query."""
ids: set[Any] = set()
for tag in tags:
tag_id = tag_id_text(tag) if isinstance(tag, dict) else None
if tag_id is None:
continue
ids.add(tag_id)
if tag_id.lstrip('-').isdigit():
ids.add(int(tag_id))
return sorted(ids, key=lambda value: (isinstance(value, str), str(value)))
# Workflow types whose pi_web_api_output_config may reference the tag database
PI_OUTPUT_WORKFLOW_TYPES = ('predictions_batch', 'xgboost_predictions_batch')
# connector_tags category each PI Web API output mapping accepts
PI_OUTPUT_CATEGORIES = {'prediction_tags': 'Write', 'confidence_tags': 'Prediction Confidence'}
# Connection defaults a pi_web_api connector may carry for the scouter query
PI_QUERY_DEFAULTS = ('period', 'max_count', 'api_timeout')
def referenced_pi_tag_ids(pipelines: list[dict[str, Any]]) -> list[Any]:
"""
The tag_ids the PI Web API pipelines reference, for the PI connector_tags query.
Read tags of pi_web_api_scouter pipelines and the prediction_tags / confidence_tags
entries of a predictions_batch pi_web_api_output_config given as a list. Same forms
and order as referenced_tag_ids.
Args:
pipelines (list[dict[str, Any]]): Pipeline configurations
Returns:
list[Any]: The referenced tag_ids (empty when no PI Web API tag carries a tag_id)
"""
tags: list[Any] = []
for pipeline in pipelines:
workflow_type = pipeline.get('workflow_type')
if workflow_type == 'pi_web_api_scouter':
read_tags = pipeline.get('read_tags')
if isinstance(read_tags, dict):
tags.extend(read_tags.values())
elif workflow_type in PI_OUTPUT_WORKFLOW_TYPES:
output_config = pipeline.get('pi_web_api_output_config') or {}
for key in PI_OUTPUT_CATEGORIES:
if isinstance(output_config.get(key), list):
tags.extend(output_config[key])
return _query_tag_ids(tags)
def filter_by_tag_ids(query: dict[str, Any], tag_ids: list[Any]) -> dict[str, Any]:
"""
Restrict a connector_tags aggregation to the referenced tag_ids.
Args:
query (dict[str, Any]): The connector_tags query ({'collection', 'aggregation'})
tag_ids (list[Any]): The ids from referenced_tag_ids
Returns:
dict[str, Any]: A new query whose aggregation starts with a tag_id $match. The
aggregation list is new too: aggregate_documents_in_mongodb appends a stage
to the list it gets.
"""
return {
**query,
'aggregation': [{'$match': {'tag_id': {'$in': tag_ids}}}, *query['aggregation']],
}
def index_connector_tags(connector_tags: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""
Index the tag database's connector_tags by tag_id.
The key is the tag_id as text, because pipelines may store it as int or str.
Documents without tag_id, connector_id, address or tag_name are left out.
Args:
connector_tags (list[dict[str, Any]]): connector_tags documents (SIENTIAPDE-1569),
each with tag_id, connector_id, address and tag_name
Returns:
dict[str, dict[str, Any]]: connector_tags documents keyed by str(tag_id)
"""
return {
str(tag['tag_id']): tag for tag in connector_tags if is_complete(tag, CONNECTOR_TAG_FIELDS)
}
def index_connectors(connectors: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""
Index the tag database's connectors by connector_id (as text).
Documents without connector_id, name, url or server_uri are left out.
Args:
connectors (list[dict[str, Any]]): connectors documents (SIENTIAPDE-1569)
Returns:
dict[str, dict[str, Any]]: connectors documents keyed by str(connector_id)
"""
return {
str(connector['connector_id']): connector
for connector in connectors
if is_complete(connector, CONNECTOR_FIELDS)
}
def resolve_tag_names(
read_tags: list[dict[str, Any]], tags_by_id: dict[str, dict[str, Any]]
) -> list[dict[str, Any]]:
"""
Fill in the tag_name of each read tag of a scouter pipeline.
The name is the pipeline's own tag_name when it has one, else the connector_tags
name of its tag_id. The scouter's model_tags (scouter()) and the slot's names[]
(gather_read_tags()) both take the name from here, so the name the gateway writes
is always the one the scouter looks for (SIENTIAPDE-1974, SC1).
Args:
read_tags (list[dict[str, Any]]): The pipeline's read_tags
tags_by_id (dict[str, dict[str, Any]]): connector_tags indexed by index_connector_tags
Returns:
list[dict[str, Any]]: The read tags, each with tag_name set when it could be
resolved. A tag_id missing from tags_by_id leaves the tag without a name.
"""
resolved = []
for tag in read_tags:
name = tag.get('tag_name')
tag_id = tag_id_text(tag)
if not name and tag_id is not None:
name = tags_by_id.get(tag_id, {}).get('tag_name')
resolved.append({**tag, 'tag_name': name} if name else tag)
return resolved
def _pi_auth(connector: dict[str, Any]) -> dict[str, Any] | None:
"""The connector's pi_web_api.auth when it names a type and a secret {name, key}."""
block = connector.get('pi_web_api')
auth = block.get('auth') if isinstance(block, dict) else None
if not isinstance(auth, dict) or auth.get('type') in (None, ''):
return None
secret = auth.get('secret')
if not isinstance(secret, dict) or not is_complete(secret, ('name', 'key')):
return None
return auth
def index_pi_connectors(connectors: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""
Index the tag database's pi_web_api connectors by connector_id (as text).
Documents without connector_id, url or a pi_web_api.auth with type and secret
{name, key} are left out, so their tags are notified as connector not found.
Args:
connectors (list[dict[str, Any]]): Active pi_web_api connectors documents
Returns:
dict[str, dict[str, Any]]: connectors documents keyed by str(connector_id)
"""
return {
str(connector['connector_id']): connector
for connector in connectors
if is_complete(connector, ('connector_id', 'url')) and _pi_auth(connector) is not None
}
def pi_connection(connector: dict[str, Any]) -> dict[str, Any]:
"""
Connection block of a pi_web_api connector for the schedule input.
Only the secret reference is copied, field by field, so nothing else the document
may hold under auth ever reaches the schedule input or the Temporal history (D3).
"""
auth = cast(dict[str, Any], _pi_auth(connector))
connection = {
'connector_id': str(connector['connector_id']),
'base_url': connector['url'],
'auth': {
'type': auth['type'],
'secret': {'name': auth['secret']['name'], 'key': auth['secret']['key']},
},
}
data_server_webid = connector['pi_web_api'].get('data_server_webid')
if data_server_webid:
connection['data_server_webid'] = data_server_webid
return connection
def _new_pi_issues() -> dict[str, set[str]]:
return {'tags': set(), 'categories': set(), 'connectors': set(), 'mixed': set()}
def _locate_pi_tag(
tag_id: str,
category: str,
pi_tags: dict[str, dict[str, Any]],
pi_connectors: dict[str, dict[str, Any]],
issues: dict[str, set[str]],
) -> dict[str, Any] | None:
"""
The connector_tags document of a PI Web API tag_id, or None when it cannot be used:
unknown tag_id ('tags'), another category ('categories'), or a connector that is not
an active, complete pi_web_api connector ('connectors').
"""
connector_tag = pi_tags.get(tag_id)
if connector_tag is None:
issues['tags'].add(tag_id)
return None
if connector_tag.get('category') != category:
issues['categories'].add(tag_id)
return None
connector_id = str(connector_tag['connector_id'])
if connector_id not in pi_connectors:
issues['connectors'].add(connector_id)
return None
return connector_tag
def resolve_pi_read_tags(
read_tags: dict[str, dict[str, Any]],
pi_tags: dict[str, dict[str, Any]],
pi_connectors: dict[str, dict[str, Any]],
) -> tuple[dict[str, dict[str, Any]], dict[str, Any] | None, dict[str, set[str]]]:
"""
Resolve the read tags of a pi_web_api_scouter pipeline through the tag database.
An entry with tag_id gets its webid from connector_tags.address (Read category); its
key (the model tag name), aggr_func and data_range stay the pipeline's. An entry
without tag_id is kept as is (legacy webid). An entry that cannot be resolved is left
out. All the resolved tags must belong to one connector (D6).
Args:
read_tags (dict[str, dict[str, Any]]): The pipeline's read_tags
pi_tags (dict[str, dict[str, Any]]): PI connector_tags indexed by index_connector_tags
pi_connectors (dict[str, dict[str, Any]]): Indexed by index_pi_connectors
Returns:
tuple: the resolved read_tags; the connector of the resolved tags (None when no tag
was resolved through the tag database or they span several connectors); and
what was left out by kind: 'tags' / 'categories' (tag_id), 'connectors'
(connector_id) and 'mixed' (the connector_ids, joined, when there are several)
"""
issues = _new_pi_issues()
resolved: dict[str, dict[str, Any]] = {}
connector_ids: set[str] = set()
for name, tag in read_tags.items():
tag_id = tag_id_text(tag)
if tag_id is None:
resolved[name] = tag
continue
connector_tag = _locate_pi_tag(tag_id, 'Read', pi_tags, pi_connectors, issues)
if connector_tag is None:
continue
connector_ids.add(str(connector_tag['connector_id']))
resolved[name] = {**tag, 'webid': connector_tag['address']}
if len(connector_ids) > 1:
issues['mixed'].add(', '.join(sorted(connector_ids)))
return resolved, None, issues
connector = pi_connectors[connector_ids.pop()] if connector_ids else None
return resolved, connector, issues
def resolve_pi_output_tags(
output_config: dict[str, Any],
pi_tags: dict[str, dict[str, Any]],
pi_connectors: dict[str, dict[str, Any]],
) -> tuple[dict[str, Any], dict[str, set[str]]]:
"""
Resolve the prediction_tags / confidence_tags of a pi_web_api_output_config.
A mapping given as a list is turned into {tag_name: WebID}: an entry with tag_id takes
tag_name and address from connector_tags ('Write' for prediction_tags, 'Prediction
Confidence' for confidence_tags); an entry without tag_id is read as a legacy
{tag_name: webid} fragment. A mapping already given as a dict passes through unchanged,
so the laborious input keeps today's shape.
Args:
output_config (dict[str, Any]): The pipeline's pi_web_api_output_config
pi_tags (dict[str, dict[str, Any]]): PI connector_tags indexed by index_connector_tags
pi_connectors (dict[str, dict[str, Any]]): Indexed by index_pi_connectors
Returns:
tuple: the resolved output config and what was left out, by kind ('tags',
'categories', 'connectors'), as in resolve_pi_read_tags
"""
issues = _new_pi_issues()
resolved = dict(output_config)
for key, category in PI_OUTPUT_CATEGORIES.items():
entries = output_config.get(key)
if not isinstance(entries, list):
continue
mapping: dict[str, Any] = {}
for entry in entries:
if not isinstance(entry, dict):
continue
tag_id = tag_id_text(entry)
if tag_id is None:
mapping.update(entry)
continue
connector_tag = _locate_pi_tag(tag_id, category, pi_tags, pi_connectors, issues)
if connector_tag is not None:
mapping[connector_tag['tag_name']] = connector_tag['address']
resolved[key] = mapping
return resolved, issues

2
orchestrator/workflows/orchestrator.py
View File

@@ -8,7 +8,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.policies import retry_policy
from orchestrator.activities.activities import Activities
from orchestrator.utils.orchestrator_functions import (
from orchestrator.utils.tag_database import (
filter_by_tag_ids,
referenced_pi_tag_ids,
referenced_tag_ids,