SIENTIAPDE-1478
Update version in init_orchestration.ipynb to 3.11.14 and enhance README.md with detailed architecture principles, execution flows, and improved filtering mechanisms in workflows. Refactor orchestrator_functions.py for clarity in filter configuration and add support for PI Web API output configuration in predictions_batch. Update tests to reflect new configurations.
This commit is contained in:
vitor-aignosi
@@ -256,17 +256,23 @@ def pi_web_api_scouter(config: dict[str, Any]):
|
||||
|
||||
def overlap_filter_config(base_filter_config: dict[str, Any], config: list[dict[str, Any]]):
|
||||
"""
|
||||
Overlap filter configuration with base filter config.
|
||||
Merge filter configurations with base filter configuration.
|
||||
|
||||
Extends the base filter configuration dictionary by adding or overwriting
|
||||
filters from the provided configuration list. Used in predictions_batch
|
||||
workflows to combine default filters with user-defined custom filters.
|
||||
|
||||
Args:
|
||||
base_filter_config (dict[str, Any]): Base filter configuration to extend.
|
||||
config (list[dict[str, Any]]): List of filter configurations to add, each containing:
|
||||
- filter_name (str): Name of the filter
|
||||
- policy (str): Filter policy
|
||||
- config (dict, optional): Additional filter configuration
|
||||
base_filter_config (dict[str, Any]): Base filter configuration dictionary to extend.
|
||||
Each filter entry contains 'policy' and optionally 'config' keys.
|
||||
config (list[dict[str, Any]]): List of filter configurations to merge, each containing:
|
||||
- filter_name (str): Name of the filter to add or update
|
||||
- policy (str): Filter policy (e.g., 'STOP', 'CONTINUE', 'REPEAT')
|
||||
- config (dict, optional): Additional filter-specific configuration
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Extended filter configuration with new filters added.
|
||||
dict[str, Any]: Extended filter configuration dictionary with merged filters.
|
||||
Filters from config list overwrite or add to base_filter_config entries.
|
||||
"""
|
||||
for fil in config:
|
||||
base_filter_config[fil['filter_name']] = {
|
||||
@@ -330,11 +336,12 @@ def predictions_batch(config: dict[str, Any]):
|
||||
- model_retention_minutes (int, optional): Data retention time in minutes (default: 60)
|
||||
- save_transform (bool, optional): Save transformed data to database (default: True)
|
||||
- predictions_storage_policy (str, optional): Prediction storage policy (default: 'lts:1')
|
||||
- pi_web_api_output_config (dict, optional): PI Web API output configuration for write-back (default: {})
|
||||
- Additional fields from common_config
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Complete predictions batch configuration with OPC output mappings,
|
||||
multi-stage filters, SQL query, and retention policies
|
||||
PI Web API output configuration, multi-stage filters, SQL query, and retention policies
|
||||
"""
|
||||
tags: dict[str, Any] = {}
|
||||
for tag in config.get('write_tags', []):
|
||||
@@ -367,6 +374,7 @@ def predictions_batch(config: dict[str, Any]):
|
||||
'transform_table_name': 'transformed_data',
|
||||
'retention_time': config.get('model_retention_minutes', 60) * 60,
|
||||
'opc_output_config': tags,
|
||||
'pi_web_api_output_config': config.get('pi_web_api_output_config', {}),
|
||||
'input_filters': overlap_filter_config(
|
||||
{'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, config.get('input_filters', [])
|
||||
),
|
||||
@@ -388,17 +396,24 @@ def predictions_batch(config: dict[str, Any]):
|
||||
|
||||
def gather_read_tags(pipelines: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""
|
||||
Gather all read tags from input pipelines.
|
||||
Gather all read tags from scouter pipeline configurations.
|
||||
|
||||
Collects all read tags from scouter pipelines and organizes them by
|
||||
server_id and tag_address, tracking which topics each tag is associated with.
|
||||
Collects all read tags from scouter-type pipelines and organizes them by
|
||||
server_id and tag_address, tracking which Kafka topics each tag is associated with.
|
||||
This function is used during slot configuration to aggregate tags across multiple
|
||||
scouter pipelines for efficient OPC server slot allocation.
|
||||
|
||||
Args:
|
||||
pipelines (list[dict[str, Any]]): The pipeline configurations to process
|
||||
pipelines (list[dict[str, Any]]): List of pipeline configurations to process.
|
||||
Only pipelines with workflow_type 'scouter' are processed. Each scouter
|
||||
pipeline should contain a 'read_tags' list with tag configurations.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Dictionary of read tags keyed by "server_id:tag_address",
|
||||
each containing tag configuration and associated topics
|
||||
where each entry contains:
|
||||
- All original tag configuration fields
|
||||
- topics (list[str]): List of Kafka topic names associated with this tag
|
||||
(format: 'raw_{schedule_name}')
|
||||
"""
|
||||
|
||||
tags = {}
|
||||
@@ -424,24 +439,31 @@ def build_tag_config(
|
||||
"""
|
||||
Build tag configuration for a specific slot and OPC server.
|
||||
|
||||
Organizes tags by OPC server and calculates the minimum subscription period
|
||||
Organizes tags by OPC server name and calculates the minimum subscription period
|
||||
based on tag frequencies. Validates that all server IDs exist in the OPC
|
||||
servers configuration.
|
||||
servers configuration. The subscription period is set to half of the minimum
|
||||
tag frequency to ensure efficient data collection.
|
||||
|
||||
Args:
|
||||
tags (list[dict[str, Any]]): List of tag configurations containing:
|
||||
- server_id (str): ID of the OPC server
|
||||
- tag_address (str): Address of the tag
|
||||
- tag_address (str): Address/path of the OPC tag
|
||||
- frequency (int): Tag read frequency in milliseconds
|
||||
opc_servers (dict[str, Any]): Dictionary of OPC server configurations
|
||||
- Additional tag-specific configuration fields
|
||||
opc_servers (dict[str, Any]): Dictionary of OPC server configurations keyed by server_id.
|
||||
Each server configuration should contain:
|
||||
- server_name (str): Human-readable server name
|
||||
- url (str): OPC server URL
|
||||
- uri (str): OPC server URI
|
||||
- cert_path (str, optional): Certificate file path
|
||||
- private_key_path (str, optional): Private key file path
|
||||
- server_cert_path (str, optional): Server certificate file path
|
||||
|
||||
Returns:
|
||||
tuple[dict[str, Any], list]: A tuple containing:
|
||||
- Slot configuration dictionary organized by server name
|
||||
- List of server IDs that were not found in opc_servers
|
||||
|
||||
Raises:
|
||||
ValueError: If the specified server_id is not found in opc_servers
|
||||
- Slot configuration dictionary organized by server_name, where each server
|
||||
contains connection details, tags dictionary, and subscription_period_ms
|
||||
- List of server IDs (str) that were not found in opc_servers configuration
|
||||
"""
|
||||
|
||||
slot_config = {}
|
||||
|
||||
Reference in New Issue
Block a user