SIENTIAPDE-1030
Add comprehensive tests and utility functions for orchestrator activities - Introduced tests for the new Formatters class, covering methods for processing schedules and slots. - Enhanced SlotManager tests with update and delete slot functionalities. - Added TemporalManager tests for creating, updating, and deleting schedules, including frequency parsing. - Implemented utility functions for orchestrator operations, including frequency parsing and tag configuration building. - Created tests for utility functions to ensure correct behavior and integration with orchestrator activities. - Established a new converters module for parsing frequency strings into seconds.
This commit is contained in:
@@ -1,42 +1,18 @@
|
||||
from os import getenv
|
||||
import json
|
||||
|
||||
|
||||
def build_postgres_config():
|
||||
def build_redis_config():
|
||||
return {
|
||||
'host': getenv('POSTGRES_HOST', 'localhost'),
|
||||
'port': int(getenv('POSTGRES_PORT', '5432')),
|
||||
'user': getenv('POSTGRES_USER', 'sientia'),
|
||||
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
||||
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
||||
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
||||
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
||||
'host': getenv('REDIS_HOST', 'localhost'),
|
||||
'port': int(getenv('REDIS_PORT', '6379')),
|
||||
'username': getenv('REDIS_USERNAME', None),
|
||||
'password': getenv('REDIS_PASSWORD', None)
|
||||
}
|
||||
|
||||
|
||||
def build_mlflow_config():
|
||||
def build_couchbase_config():
|
||||
return {
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config():
|
||||
opc_raw = getenv('OPC_CONFIG', None)
|
||||
|
||||
if opc_raw:
|
||||
return json.loads(opc_raw)
|
||||
|
||||
return {
|
||||
'opc': {
|
||||
'name': getenv('OPC_NAME', 'opc'),
|
||||
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
||||
'cert_path': getenv('OPC_CERT_PATH', None),
|
||||
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
||||
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
|
||||
}
|
||||
'connection_string': getenv('COUCHBASE_CONNECTION_STRING', 'couchbase://localhost'),
|
||||
'username': getenv('COUCHBASE_USERNAME', 'sientia'),
|
||||
'password': getenv('COUCHBASE_PASSWORD', 'sientia')
|
||||
}
|
||||
|
||||
14
orchestrator/utils/converters.py
Normal file
14
orchestrator/utils/converters.py
Normal file
@@ -0,0 +1,14 @@
|
||||
def parse_frequency(frequency: str) -> int:
|
||||
"""
|
||||
Parse frequency string to seconds
|
||||
"""
|
||||
if frequency.endswith("s"):
|
||||
return int(frequency[:-1])
|
||||
elif frequency.endswith("m"):
|
||||
return int(frequency[:-1]) * 60
|
||||
elif frequency.endswith("h"):
|
||||
return int(frequency[:-1]) * 60 * 60
|
||||
elif frequency.endswith("d"):
|
||||
return int(frequency[:-1]) * 60 * 60 * 24
|
||||
else:
|
||||
raise ValueError("Invalid frequency")
|
||||
162
orchestrator/utils/orchestrator_functions.py
Normal file
162
orchestrator/utils/orchestrator_functions.py
Normal file
@@ -0,0 +1,162 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
def common_config(config: dict[str, Any]):
|
||||
return {
|
||||
"workflow_type": config['workflow_type'],
|
||||
"schedule_name": config['schedule_name'],
|
||||
"frequency": config.get('frequency', '1m'),
|
||||
"max_retry_policy": config.get('max_retry_policy', 1),
|
||||
|
||||
"model_id": config['model_id'],
|
||||
"model_name": config['model_name'],
|
||||
}
|
||||
|
||||
|
||||
def scouter(config: dict[str, Any]):
|
||||
filters = {}
|
||||
for f in config.get('filters', []):
|
||||
filters[f['filter_name']] = {
|
||||
"policy": f['policy']
|
||||
}
|
||||
|
||||
tags = {}
|
||||
for tag in config['read_tags']:
|
||||
tags[tag['tag_name']] = {
|
||||
"aggr_func": tag.get('aggr_func', 'lts'),
|
||||
"data_range": tag.get('data_range', [-100, 100])
|
||||
}
|
||||
|
||||
return {
|
||||
**common_config(config),
|
||||
|
||||
"topic": f"raw_{config['schedule_name']}",
|
||||
"trigger_laborious": False,
|
||||
"filters": filters,
|
||||
"schema": "sientia_data",
|
||||
"table_name": "laborious_data",
|
||||
"retention_time": config.get('tag_retention_minutes', 60) * 60,
|
||||
"model_tags": tags
|
||||
}
|
||||
|
||||
|
||||
def overlap_filter_config(base_filter_config: dict[str, Any], config: list[dict[str, Any]]):
|
||||
for fil in config:
|
||||
base_filter_config[fil['filter_name']] = {
|
||||
"policy": fil['policy'],
|
||||
"config": fil.get('config', {})
|
||||
}
|
||||
|
||||
return base_filter_config
|
||||
|
||||
|
||||
def process_path_priority(path_priority: list[str]):
|
||||
for priority in path_priority[:]:
|
||||
if priority not in ["STOP", "CONTINUE", "REPEAT"]:
|
||||
path_priority.remove(priority)
|
||||
|
||||
for priority in ["STOP", "CONTINUE", "REPEAT"]:
|
||||
if priority not in path_priority:
|
||||
path_priority.append(priority)
|
||||
|
||||
return path_priority[0:3]
|
||||
|
||||
|
||||
def predictions_batch(config: dict[str, Any]):
|
||||
tags = {}
|
||||
for tag in config['write_tags']:
|
||||
if tag['server_name'] not in tags:
|
||||
tags[tag['server_name']] = {}
|
||||
|
||||
tag_type = tag['type']
|
||||
|
||||
if tag_type == 'prediction' or tag_type == 'confidence':
|
||||
tag_type_str = f"{tag_type}_tags"
|
||||
|
||||
if tag_type_str not in tags[tag['server_name']]:
|
||||
tags[tag['server_name']][tag_type_str] = {}
|
||||
|
||||
tags[tag['server_name']][tag_type_str][tag['addr']] = {
|
||||
"data_type": tag.get('data_type', 'float'),
|
||||
}
|
||||
|
||||
path_priority = process_path_priority(config.get(
|
||||
'path_priority', ["STOP", "CONTINUE", "REPEAT"]))
|
||||
|
||||
return {
|
||||
**common_config(config),
|
||||
|
||||
"query": config['query'],
|
||||
"schema": "sientia_data",
|
||||
"table_name": "predictions",
|
||||
"retention_time": config.get('model_retention_minutes', 60) * 60,
|
||||
"opc_output_config": tags,
|
||||
"input_filters": overlap_filter_config({
|
||||
"EMPTY_DATA": {
|
||||
"policy": "STOP",
|
||||
"config": {}
|
||||
}
|
||||
}, config['input_filters']),
|
||||
"mlflow_transform_filters": overlap_filter_config({
|
||||
"API_ERROR": {
|
||||
"policy": "STOP",
|
||||
"config": {}
|
||||
}
|
||||
}, config['mlflow_transform_filters']),
|
||||
"mlflow_predict_filters": overlap_filter_config({
|
||||
"API_ERROR": {
|
||||
"policy": "STOP",
|
||||
"config": {}
|
||||
}
|
||||
}, config['mlflow_predict_filters']),
|
||||
"path_priority": path_priority
|
||||
}
|
||||
|
||||
|
||||
def gather_read_tags(pipelines: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""
|
||||
Gathers all read tags from input pipelines.
|
||||
|
||||
Args:
|
||||
- pipelines (list[dict[str, Any]]): The schedules to process.
|
||||
|
||||
Returns:
|
||||
- dict[str, Any]: The read tags dictionary
|
||||
"""
|
||||
|
||||
tags = {}
|
||||
|
||||
# Get all read tags from pipelines
|
||||
for pipeline in pipelines:
|
||||
for tag in pipeline['read_tags']:
|
||||
tag_string = f"{tag['server_name']}:{tag['tag_address']}"
|
||||
if tag_string not in tags:
|
||||
tags[tag_string] = {
|
||||
**tag,
|
||||
"topics": []
|
||||
}
|
||||
|
||||
tags[tag_string]['topics'].append(
|
||||
f"raw_{pipeline['schedule_name']}")
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
def build_tag_config(tag: dict[str, Any], slot_config: dict[str, Any],
|
||||
opc_servers: dict[str, Any], i: int):
|
||||
server_name = tag['server_name']
|
||||
if server_name not in slot_config[f"{i}"]:
|
||||
slot_config[f"{i}"][server_name] = {
|
||||
"name": server_name,
|
||||
"url": opc_servers[server_name]['url'],
|
||||
"server_uri": opc_servers[server_name]['uri'],
|
||||
"tags": {}
|
||||
}
|
||||
for name, spec in opc_servers[server_name].get('security_spec', {}).items():
|
||||
slot_config[f"{i}"][server_name][name] = spec
|
||||
|
||||
slot_config[f"{i}"][server_name]["tags"][tag['tag_address']] = {
|
||||
**tag,
|
||||
}
|
||||
|
||||
return slot_config
|
||||
Reference in New Issue
Block a user