gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Refactor orchestrator activities and configuration files

- Removed unused Temporal timeout environment variables from values.yaml.
- Reorganized import statements in various activity files for better readability.
- Updated logging messages to use consistent formatting across activities.
- Enhanced test cases to ensure proper initialization and shutdown of orchestrator activities.
- Improved overall code structure and readability by applying consistent formatting and style adjustments.
This commit is contained in:
vitor-aignosi
2025-10-15 17:20:52 -03:00
parent 310cae4c2f
commit f3a6af5603
39 changed files with 4988 additions and 3826 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
orchestrator/utils/connectors_config.py
View File

@@ -12,7 +12,7 @@ def build_redis_config():
'host': getenv('REDIS_HOST', 'localhost'),
'port': int(getenv('REDIS_PORT', '6379')),
'username': getenv('REDIS_USERNAME', 'default'),
'password': getenv('REDIS_PASSWORD', 'bdnZOpcyiL')
'password': getenv('REDIS_PASSWORD', 'bdnZOpcyiL'),
}
@@ -31,7 +31,7 @@ def build_mongodb_config():
return {
'connection_string': connection_string,
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
}
@@ -45,7 +45,7 @@ def build_couchbase_config():
return {
'connection_string': getenv('COUCHBASE_CONNECTION_STRING', 'couchbase://localhost'),
'username': getenv('COUCHBASE_USERNAME', 'sientia'),
'password': getenv('COUCHBASE_PASSWORD', 'sientia')
'password': getenv('COUCHBASE_PASSWORD', 'sientia'),
}
@@ -61,9 +61,6 @@ def build_temporal_config():
'temporal_namespace': getenv('TEMPORAL_NAMESPACE', 'default'),
'temporal_scouter_namespace': getenv('TEMPORAL_SCOUTER_NAMESPACE', 'scouter'),
'temporal_laborious_namespace': getenv('TEMPORAL_LABORIOUS_NAMESPACE', 'laborious'),
'temporal_task_timeout_minutes': int(getenv('TEMPORAL_TASK_TIMEOUT_MINUTES', '5')),
'temporal_run_timeout_minutes': int(getenv('TEMPORAL_RUN_TIMEOUT_MINUTES', '5')),
'temporal_execution_timeout_minutes': int(getenv('TEMPORAL_EXECUTION_TIMEOUT_MINUTES', '5'))
}
@@ -81,7 +78,7 @@ def build_postgres_config():
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')),
}
@@ -96,5 +93,5 @@ def build_email_config():
'sender_email': getenv('EMAIL_SENDER', 'sientia-alerts@aignosi.com'),
'sender_password': getenv('EMAIL_SENDER_PASSWORD', 'sientia'),
'smtp_server': getenv('EMAIL_SMTP_SERVER', None),
'smtp_port': int(getenv('EMAIL_SMTP_PORT', '587'))
'smtp_port': int(getenv('EMAIL_SMTP_PORT', '587')),
}

12
orchestrator/utils/converters.py
View File

@@ -9,7 +9,7 @@ def parse_frequency(frequency: str) -> int:
Args:
frequency (str): Frequency string with suffix:
- 's' for seconds (e.g., '30s')
- 'm' for minutes (e.g., '5m')
- 'm' for minutes (e.g., '5m')
- 'h' for hours (e.g., '2h')
- 'd' for days (e.g., '1d')
@@ -19,13 +19,13 @@ def parse_frequency(frequency: str) -> int:
Raises:
ValueError: If frequency format is invalid
"""
if frequency.endswith("s"):
if frequency.endswith('s'):
return int(frequency[:-1])
elif frequency.endswith("m"):
elif frequency.endswith('m'):
return int(frequency[:-1]) * 60
elif frequency.endswith("h"):
elif frequency.endswith('h'):
return int(frequency[:-1]) * 60 * 60
elif frequency.endswith("d"):
elif frequency.endswith('d'):
return int(frequency[:-1]) * 60 * 60 * 24
else:
raise ValueError("Invalid frequency")
raise ValueError('Invalid frequency')

54
orchestrator/utils/email_builder.py
View File

@@ -1,8 +1,5 @@
import json
from sientia_do.observability.logger import Logger
from sientia_do.notifications.models import NotificationLevel
from jinja2 import Template
import re
from sientia_do.observability.logger import Logger
class EmailBuilder:
@@ -24,9 +21,9 @@ class EmailBuilder:
self.report_template_file = './orchestrator/utils/templates/email_template.html'
self.general_template_file = './orchestrator/utils/templates/general_template.html'
with open(self.report_template_file, 'r') as file:
with open(self.report_template_file) as file:
self.report_template = file.read()
with open(self.general_template_file, 'r') as file:
with open(self.general_template_file) as file:
self.general_template = file.read()
def replace_parameters(self, template: str, parameters: dict) -> str:
@@ -41,9 +38,9 @@ class EmailBuilder:
str: The rendered template with parameters replaced.
"""
# Criar um template Jinja2
template = Template(template)
template_obj = Template(template)
return template.render(parameters)
return template_obj.render(parameters)
def parameters(self, general_events: dict, mail_type: str) -> dict:
"""
@@ -57,21 +54,25 @@ class EmailBuilder:
Returns:
dict: Dictionary with mail_type and rendered event sections for each notification level.
"""
error_models = general_events.get('ERROR', {}).get('models', [])
warning_models = general_events.get('WARNING', {}).get('models', [])
info_models = general_events.get('INFO', {}).get('models', [])
error_events = general_events.get('ERROR', {})
warning_events = general_events.get('WARNING', {})
info_events = general_events.get('INFO', {})
error_models = error_events.get('models', [])
warning_models = warning_events.get('models', [])
info_models = info_events.get('models', [])
return {
'mail_type': mail_type,
'error_events': self.replace_parameters(self.general_template,
general_events.get(
'ERROR')) if error_models else '',
'warning_events': self.replace_parameters(self.general_template,
general_events.get(
'WARNING')) if warning_models else '',
'info_events': self.replace_parameters(self.general_template,
general_events.get(
'INFO')) if info_models else '',
'error_events': self.replace_parameters(self.general_template, error_events)
if error_models
else '',
'warning_events': self.replace_parameters(self.general_template, warning_events)
if warning_models
else '',
'info_events': self.replace_parameters(self.general_template, info_events)
if info_models
else '',
}
def build_email(self, report_data: list[dict], mail_type: str) -> str:
@@ -90,29 +91,26 @@ class EmailBuilder:
general_events = {}
for report in report_data:
level = report['level']
model_name = report['model_name']
if level not in general_events:
general_events[level] = {
'section_name': f'{level.capitalize()}s detected:',
'models': {}
'models': {},
}
if model_name not in general_events[level]['models']:
general_events[level]['models'][model_name] = {
'model_name': model_name,
'events': []
'events': [],
}
general_events[level]['models'][model_name]['events'].append(
report)
general_events[level]['models'][model_name]['events'].append(report)
for _type, content in general_events.items():
content['models'] = list(content['models'].values())
return self.replace_parameters(
self.report_template, self.parameters(
general_events, mail_type
))
self.report_template, self.parameters(general_events, mail_type)
)

166
orchestrator/utils/orchestrator_functions.py
View File

@@ -19,17 +19,15 @@ def common_config(config: dict[str, Any]):
"""
model = config['model']
return {
"workflow_type": config['workflow_type'],
"schedule_name": config['schedule_name'],
"frequency": config.get('frequency', '1m'),
"max_retry_policy": config.get('max_retry_policy', 1),
"model_id": config['model_id'],
"model_name": model['name'],
"model_config": model.get('model_config', {}),
"execution_timeout_seconds": config.get('execution_timeout_seconds', 300),
"task_timeout_seconds": config.get('task_timeout_seconds', 300),
'workflow_type': config['workflow_type'],
'schedule_name': config['schedule_name'],
'frequency': config.get('frequency', '1m'),
'max_retry_policy': config.get('max_retry_policy', 1),
'model_id': config['model_id'],
'model_name': model['name'],
'model_config': model.get('model_config', {}),
'execution_timeout_seconds': config.get('execution_timeout_seconds', 300),
'task_timeout_seconds': config.get('task_timeout_seconds', 300),
}
@@ -48,12 +46,12 @@ def minimal_retrain(config: dict[str, Any]):
"""
return {
**common_config(config),
"workflow_type": "minimal_retrain",
"schedule_name": config['schedule_name'],
"query": config['query'],
"schema": "sientia_data",
"table_name": "log_retrain",
"datetime_columns": config.get('datetime_columns', []),
'workflow_type': 'minimal_retrain',
'schedule_name': config['schedule_name'],
'query': config['query'],
'schema': 'sientia_data',
'table_name': 'log_retrain',
'datetime_columns': config.get('datetime_columns', []),
}
@@ -79,28 +77,25 @@ def scouter(config: dict[str, Any]):
"""
filters = {}
for f in config.get('filters', []):
filters[f['filter_name']] = {
"policy": f['policy']
}
filters[f['filter_name']] = {'policy': f['policy']}
tags = {}
for tag in config['read_tags']:
tags[tag['tag_name']] = {
"aggr_func": tag.get('aggr_func', 'lts'),
"data_range": tag.get('data_range', [-100, 100])
'aggr_func': tag.get('aggr_func', 'lts'),
'data_range': tag.get('data_range', [-100, 100]),
}
return {
**common_config(config),
"topic": f"raw_{config['schedule_name']}",
"trigger_laborious": False,
"filters": filters,
"schema": "sientia_data",
"table_name": "laborious_data",
"retention_time": config.get('tag_retention_minutes', 60) * 60,
"model_tags": tags,
"debug_data_package": config.get('debug_data_package', False)
'topic': f'raw_{config["schedule_name"]}',
'trigger_laborious': False,
'filters': filters,
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': config.get('tag_retention_minutes', 60) * 60,
'model_tags': tags,
'debug_data_package': config.get('debug_data_package', False),
}
@@ -120,8 +115,8 @@ def overlap_filter_config(base_filter_config: dict[str, Any], config: list[dict[
"""
for fil in config:
base_filter_config[fil['filter_name']] = {
"policy": fil['policy'],
"config": fil.get('config', {})
'policy': fil['policy'],
'config': fil.get('config', {}),
}
return base_filter_config
@@ -138,10 +133,10 @@ def process_path_priority(path_priority: list[str]):
list[str]: Normalized path priority list with exactly 3 elements: ["STOP", "CONTINUE", "REPEAT"].
"""
for priority in path_priority[:]:
if priority not in ["STOP", "CONTINUE", "REPEAT"]:
if priority not in ['STOP', 'CONTINUE', 'REPEAT']:
path_priority.remove(priority)
for priority in ["STOP", "CONTINUE", "REPEAT"]:
for priority in ['STOP', 'CONTINUE', 'REPEAT']:
if priority not in path_priority:
path_priority.append(priority)
@@ -169,7 +164,7 @@ def predictions_batch(config: dict[str, Any]):
Returns:
dict[str, Any]: Predictions batch configuration with OPC output config, filters, and path priority.
"""
tags = {}
tags: dict[str, Any] = {}
for tag in config.get('write_tags', []):
if tag['server_id'] not in tags:
tags[tag['server_id']] = {}
@@ -177,51 +172,43 @@ def predictions_batch(config: dict[str, Any]):
tag_type = tag['type']
if tag_type == 'prediction' or tag_type == 'confidence':
tag_type_str = f"{tag_type}_tags"
tag_type_str = f'{tag_type}_tags'
if tag_type_str not in tags[tag['server_id']]:
tags[tag['server_id']][tag_type_str] = {}
tags[tag['server_id']][tag_type_str][tag['addr']] = {
"data_type": tag.get('data_type', 'float'),
'data_type': tag.get('data_type', 'float'),
}
path_priority = process_path_priority(config.get(
'path_priority', ["STOP", "CONTINUE", "REPEAT"]))
path_priority = process_path_priority(
config.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT'])
)
return {
**common_config(config),
"query": config['query'],
"datetime_columns": config.get('datetime_columns', []),
"schema": "sientia_data",
"table_name": "predictions",
"retention_time": config.get('model_retention_minutes', 60) * 60,
"opc_output_config": tags,
"input_filters": overlap_filter_config({
"EMPTY_DATA": {
"policy": "STOP",
"config": {}
}
}, config.get('input_filters', [])),
"mlflow_transform_filters": overlap_filter_config({
"EMPTY_DATA": {
"policy": "STOP",
"config": {}
'query': config['query'],
'datetime_columns': config.get('datetime_columns', []),
'schema': 'sientia_data',
'table_name': 'predictions',
'retention_time': config.get('model_retention_minutes', 60) * 60,
'opc_output_config': tags,
'input_filters': overlap_filter_config(
{'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, config.get('input_filters', [])
),
'mlflow_transform_filters': overlap_filter_config(
{
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
"API_ERROR": {
"policy": "STOP",
"config": {}
}
}, config.get('mlflow_transform_filters', [])),
"mlflow_predict_filters": overlap_filter_config({
"API_ERROR": {
"policy": "STOP",
"config": {}
}
}, config.get('mlflow_predict_filters', [])),
"path_priority": path_priority,
"predictions_storage_policy": config.get('predictions_storage_policy', 'lts:1')
config.get('mlflow_transform_filters', []),
),
'mlflow_predict_filters': overlap_filter_config(
{'API_ERROR': {'policy': 'STOP', 'config': {}}},
config.get('mlflow_predict_filters', []),
),
'path_priority': path_priority,
'predictions_storage_policy': config.get('predictions_storage_policy', 'lts:1'),
}
@@ -241,21 +228,18 @@ def gather_read_tags(pipelines: list[dict[str, Any]]) -> dict[str, Any]:
# Get all read tags from pipelines
for pipeline in pipelines:
for tag in pipeline.get('read_tags', []):
tag_string = f"{tag['server_id']}:{tag['tag_address']}"
tag_string = f'{tag["server_id"]}:{tag["tag_address"]}'
if tag_string not in tags:
tags[tag_string] = {
**tag,
"topics": []
}
tags[tag_string] = {**tag, 'topics': []}
tags[tag_string]['topics'].append(
f"raw_{pipeline['schedule_name']}")
tags[tag_string]['topics'].append(f'raw_{pipeline["schedule_name"]}')
return tags
def build_tag_config(tag: dict[str, Any], slot_config: dict[str, Any],
opc_servers: dict[str, Any], i: int):
def build_tag_config(
tag: dict[str, Any], slot_config: dict[str, Any], opc_servers: dict[str, Any], i: int
):
"""
Build tag configuration for a specific slot and OPC server.
@@ -276,22 +260,22 @@ def build_tag_config(tag: dict[str, Any], slot_config: dict[str, Any],
server_id = tag['server_id']
if server_id not in opc_servers:
raise ValueError(f"Server {server_id} not found in opc_servers")
raise ValueError(f'Server {server_id} not found in opc_servers')
server_name = opc_servers[server_id]['server_name']
if server_name not in slot_config[f"{i}"]:
slot_config[f"{i}"][server_name] = {
"server_id": server_id,
"name": server_name,
"url": opc_servers[server_id]['url'],
"server_uri": opc_servers[server_id]['uri'],
"cert_path": opc_servers[server_id].get('cert_path', None),
"private_key_path": opc_servers[server_id].get('private_key_path', None),
"server_cert_path": opc_servers[server_id].get('server_cert_path', None),
"tags": {}
if server_name not in slot_config[f'{i}']:
slot_config[f'{i}'][server_name] = {
'server_id': server_id,
'name': server_name,
'url': opc_servers[server_id]['url'],
'server_uri': opc_servers[server_id]['uri'],
'cert_path': opc_servers[server_id].get('cert_path', None),
'private_key_path': opc_servers[server_id].get('private_key_path', None),
'server_cert_path': opc_servers[server_id].get('server_cert_path', None),
'tags': {},
}
slot_config[f"{i}"][server_name]["tags"][tag['tag_address']] = {
slot_config[f'{i}'][server_name]['tags'][tag['tag_address']] = {
**tag,
}