Code import - branch feature/SIENTIAPDE-1646

orchestrator/utils/connectors_config.py

@@ -0,0 +1,97 @@
+from os import getenv
+
+
+def build_redis_config():
+    """
+    Build Redis configuration from environment variables.
+
+    Returns:
+        dict: Redis configuration with host, port, username, and password.
+    """
+    return {
+        'host': getenv('REDIS_HOST', 'localhost'),
+        'port': int(getenv('REDIS_PORT', '6379')),
+        'username': getenv('REDIS_USERNAME', 'default'),
+        'password': getenv('REDIS_PASSWORD', 'bdnZOpcyiL'),
+    }
+
+
+def build_mongodb_config():
+    """
+    Build MongoDB configuration from environment variables.
+
+    Returns:
+        dict: MongoDB configuration with connection string, database name, and TTL index seconds.
+    """
+    username = getenv('MONGODB_USERNAME', 'root')
+    password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
+    uri = getenv('MONGODB_URL', 'localhost:27018')
+
+    connection_string = f'mongodb://{username}:{password}@{uri}'
+    return {
+        'connection_string': connection_string,
+        'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
+        'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
+    }
+
+
+def build_couchbase_config():
+    """
+    Build Couchbase configuration from environment variables.
+
+    Returns:
+        dict: Couchbase configuration with connection string, username, and password.
+    """
+    return {
+        'connection_string': getenv('COUCHBASE_CONNECTION_STRING', 'couchbase://localhost'),
+        'username': getenv('COUCHBASE_USERNAME', 'sientia'),
+        'password': getenv('COUCHBASE_PASSWORD', 'sientia'),
+    }
+
+
+def build_temporal_config():
+    """
+    Build Temporal configuration from environment variables.
+
+    Returns:
+        dict: Temporal configuration with host and namespace settings.
+    """
+    return {
+        'temporal_host': getenv('TEMPORAL_HOST', 'localhost:7233'),
+        'temporal_namespace': getenv('TEMPORAL_NAMESPACE', 'default'),
+        'temporal_scouter_namespace': getenv('TEMPORAL_SCOUTER_NAMESPACE', 'scouter'),
+        'temporal_laborious_namespace': getenv('TEMPORAL_LABORIOUS_NAMESPACE', 'laborious'),
+    }
+
+
+def build_postgres_config():
+    """
+    Build PostgreSQL configuration from environment variables.
+
+    Returns:
+        dict: PostgreSQL configuration with connection details and connection pool settings.
+    """
+    return {
+        'host': getenv('POSTGRES_HOST', 'localhost'),
+        'port': int(getenv('POSTGRES_PORT', '5432')),
+        'user': getenv('POSTGRES_USER', 'sientia'),
+        'password': getenv('POSTGRES_PASSWORD', 'sientia'),
+        'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
+        'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
+        'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')),
+    }
+
+
+def build_email_config():
+    """
+    Build email configuration from environment variables.
+
+    Returns:
+        dict: Email configuration with SMTP server settings and sender credentials.
+    """
+    return {
+        'sender_email': getenv('EMAIL_SENDER', 'sientia-alerts@aignosi.com'),
+        'sender_password': getenv('EMAIL_SENDER_PASSWORD', 'sientia'),
+        'smtp_server': getenv('EMAIL_SMTP_SERVER', None),
+        'smtp_port': int(getenv('EMAIL_SMTP_PORT', '587')),
+    }

orchestrator/utils/converters.py

@@ -0,0 +1,31 @@
+def parse_frequency(frequency: str) -> int:
+    """
+    Parse frequency string into seconds for Temporal schedule intervals.
+
+    This function converts human-readable frequency strings into seconds
+    for use in Temporal schedule configurations. Supports seconds, minutes,
+    hours, and days notation.
+
+    Args:
+        frequency (str): Frequency string with suffix:
+            - 's' for seconds (e.g., '30s')
+            - 'm' for minutes (e.g., '5m')
+            - 'h' for hours (e.g., '2h')
+            - 'd' for days (e.g., '1d')
+
+    Returns:
+        int: Frequency converted to seconds
+
+    Raises:
+        ValueError: If frequency format is invalid
+    """
+    if frequency.endswith('s'):
+        return int(frequency[:-1])
+    elif frequency.endswith('m'):
+        return int(frequency[:-1]) * 60
+    elif frequency.endswith('h'):
+        return int(frequency[:-1]) * 60 * 60
+    elif frequency.endswith('d'):
+        return int(frequency[:-1]) * 60 * 60 * 24
+    else:
+        raise ValueError('Invalid frequency')

orchestrator/utils/email_builder.py

@@ -0,0 +1,132 @@
+from typing import Any
+
+from jinja2 import Template
+from sientia_do.observability.logger import Logger
+
+
+class EmailBuilder:
+    """
+    HTML email template builder for notification emails.
+
+    This class handles the generation of HTML email content from notification
+    data using Jinja2 templates. It supports different email types (alerts,
+    reports) and notification levels (ERROR, WARNING, INFO) with customizable
+    templates and parameter replacement.
+
+    Args:
+        logger (Logger): Application logger instance for error reporting
+    """
+
+    def __init__(self, logger: Logger):
+        self.logger = logger
+
+        self.report_template_file = './orchestrator/utils/templates/email_template.html'
+        self.general_template_file = './orchestrator/utils/templates/general_template.html'
+
+        with open(self.report_template_file) as file:
+            self.report_template = file.read()
+        with open(self.general_template_file) as file:
+            self.general_template = file.read()
+
+    def replace_parameters(self, template: str, parameters: dict) -> str:
+        """
+        Replace parameters in a Jinja2 template with provided values.
+
+        Renders a Jinja2 template string with the provided parameter dictionary,
+        replacing all template variables with their corresponding values.
+
+        Args:
+            template (str): The Jinja2 template string
+            parameters (dict): Dictionary of parameters to replace in the template
+
+        Returns:
+            str: The rendered template with parameters replaced
+        """
+        # Create a Jinja2 template from the provided string
+        template_obj = Template(template)
+
+        return template_obj.render(parameters)
+
+    def parameters(self, general_events: dict, mail_type: str) -> dict:
+        """
+        Build parameters dictionary for email templates based on general events and mail type.
+
+        Processes notification events organized by level and model, rendering HTML
+        sections for each notification level using the general template.
+
+        Args:
+            general_events (dict): Dictionary containing events categorized by level (ERROR, WARNING, INFO).
+                Each level contains a 'models' key with model-specific event data
+            mail_type (str): The type of email being sent (Alerts/Reports)
+
+        Returns:
+            dict: Dictionary with mail_type and rendered event sections for each notification level
+        """
+        error_events = general_events.get('ERROR', {})
+        warning_events = general_events.get('WARNING', {})
+        info_events = general_events.get('INFO', {})
+
+        error_models = error_events.get('models', [])
+        warning_models = warning_events.get('models', [])
+        info_models = info_events.get('models', [])
+
+        return {
+            'mail_type': mail_type,
+            'error_events': self.replace_parameters(self.general_template, error_events)
+            if error_models
+            else '',
+            'warning_events': self.replace_parameters(self.general_template, warning_events)
+            if warning_models
+            else '',
+            'info_events': self.replace_parameters(self.general_template, info_events)
+            if info_models
+            else '',
+        }
+
+    def build_email(self, report_data: list[dict[str, Any]], mail_type: str) -> str:
+        """
+        Build the email HTML by organizing report data by notification level and model.
+
+        Organizes notification data by level and model, then renders the complete
+        HTML email using the report template with all event sections.
+
+        Args:
+            report_data (list[dict[str, Any]]): List of notification reports, each containing:
+                - level (str): Notification level (ERROR, WARNING, INFO)
+                - model_name (str): Name of the model
+                - Additional notification details
+            mail_type (str): The type of email being built (Alerts/Reports)
+
+        Returns:
+            str: Complete HTML email content ready for sending
+        """
+        general_events: dict[str, dict[str, Any]] = {}
+
+        for report in report_data:
+            level = report['level']
+            model_name = report['model_name']
+
+            if level not in general_events:
+                general_events[level] = {
+                    'section_name': f'{level.capitalize()}s detected:',
+                    'models': {},
+                }
+
+            # Type assertion to help the type checker understand the structure
+            level_data = general_events[level]
+            models_dict = level_data['models']
+
+            if model_name not in models_dict:
+                models_dict[model_name] = {
+                    'model_name': model_name,
+                    'events': [],
+                }
+
+            models_dict[model_name]['events'].append(report)
+
+        for _type, content in general_events.items():
+            content['models'] = list(content['models'].values())
+
+        return self.replace_parameters(
+            self.report_template, self.parameters(general_events, mail_type)
+        )

orchestrator/utils/orchestrator_functions.py

@@ -0,0 +1,505 @@
+from typing import Any
+
+from orchestrator.utils.converters import parse_frequency
+
+
+def common_config(config: dict[str, Any]):
+    """
+    Extract common configuration parameters from a pipeline configuration.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - workflow_type (str): Type of workflow (e.g., 'scouter', 'predictions_batch', 'drift')
+            - schedule_name (str): Unique name identifier for this schedule
+            - model_id (str): MongoDB ID of the associated model
+            - model (dict): Model configuration containing:
+                - name (str): Human-readable name of the model
+                - model_config (dict, optional): Additional model-specific configuration
+            - frequency (str, optional): Execution frequency (default: '1m')
+                Format: '{number}{unit}' where unit is 's', 'm', 'h', or 'd'
+            - offset (str, optional): Schedule offset/delay (default: '0m')
+            - max_retry_policy (int, optional): Maximum retry attempts on failure (default: 1)
+            - execution_timeout_seconds (int, optional): Workflow execution timeout (default: 300)
+            - task_timeout_seconds (int, optional): Individual task timeout (default: 300)
+
+    Returns:
+        dict[str, Any]: Common configuration dictionary with standardized parameters
+            for Temporal workflow execution
+    """
+    model = config['model']
+    return {
+        'workflow_type': config['workflow_type'],
+        'schedule_name': config['schedule_name'],
+        'frequency': config.get('frequency', '1m'),
+        'offset': config.get('offset', '0m'),
+        'max_retry_policy': config.get('max_retry_policy', 1),
+        'model_id': config['model_id'],
+        'model_name': model['name'],
+        'model_config': model.get('model_config', {}),
+        'execution_timeout_seconds': config.get('execution_timeout_seconds', 300),
+        'task_timeout_seconds': config.get('task_timeout_seconds', 300),
+        'on_conflict': config.get('on_conflict', 'error'),
+        'runtime': config.get('runtime', 'legacy'),
+    }
+
+
+def drift(config: dict[str, Any]):
+    """
+    Build drift configuration from pipeline config.
+
+    Creates a drift detection workflow configuration with database table mappings
+    and drift metric specifications for monitoring data distribution changes.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - interval_minutes (int, optional): Time window for data comparison in minutes (default: 60)
+            - drift_metrics (list[str], optional): Statistical metrics to compute (default:
+                ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein'])
+            - Additional fields from common_config
+
+    Returns:
+        dict[str, Any]: Drift detection configuration with source/target tables,
+            time interval, and metrics specifications. Results stored in 'drift_metrics' table
+    """
+    return {
+        **common_config(config),
+        'schema': 'sientia_data',
+        'source_table_name': 'laborious_data',
+        'target_table_name': 'drift_metrics',
+        'interval': config.get('interval_minutes', 60),
+        'drift_metrics': config.get(
+            'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
+        ),
+    }
+
+
+def simple_metrics(config: dict[str, Any]):
+    """
+    Build simple metrics configuration from pipeline config.
+
+    Creates a simple metrics computation workflow configuration for calculating
+    model performance metrics like RMSE, MSE, MAE, and R².
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - interval_minutes (int, optional): Computation interval in minutes (default: 60)
+            - metrics (list[str], optional): List of metrics to compute
+                (default: ['rmse', 'mse', 'mae', 'r2'])
+            - Additional fields from common_config
+
+    Returns:
+        dict[str, Any]: Simple metrics configuration with source tables (predictions and
+            actual data), target table, time interval, and metrics list. Results stored in
+            'simple_metrics' table
+    """
+    return {
+        **common_config(config),
+        'schema': 'sientia_data',
+        'predictions_table_name': 'predictions',
+        'data_table_name': 'laborious_data',
+        'target_table_name': 'simple_metrics',
+        'interval_minutes': config.get('interval_minutes', 60),
+        'metrics': config.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
+    }
+
+
+def minimal_retrain(config: dict[str, Any]):
+    """
+    Build minimal retrain configuration from pipeline config.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - query (str): SQL query to retrieve training data. Should return features
+                and target variable in expected format
+            - datetime_columns (list[str], optional): Column names to parse as datetime
+                for proper temporal handling (default: [])
+            - Additional fields from common_config
+
+    Returns:
+        dict[str, Any]: Minimal retrain configuration with SQL query, database settings,
+            and datetime column specifications. Retraining logs are stored in 'log_retrain' table
+    """
+    return {
+        **common_config(config),
+        'query': config['query'],
+        'schema': 'sientia_data',
+        'table_name': 'log_retrain',
+        'datetime_columns': config.get('datetime_columns', []),
+    }
+
+
+def base_scouter(config: dict[str, Any]):
+    """
+    Build base scouter configuration shared by all scouter workflow types.
+
+    Creates the foundational configuration for OPC data collection workflows,
+    including filter policies, database settings, and data retention parameters.
+    This configuration is extended by specific scouter implementations (OPC UA, PI Web API).
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - filters (list[dict], optional): List of filter configurations with:
+                - filter_name (str): Name of the filter
+                - policy (str): Filter policy to apply
+            - tag_retention_minutes (int, optional): Tag retention time in minutes (default: 60)
+            - debug_data_package (bool, optional): Enable debug data package logging (default: False)
+            - fill_missing_tags (bool, optional): Fill missing tags with interpolation (default: False)
+            - Additional fields from common_config
+
+    Returns:
+        dict[str, Any]: Base scouter configuration with filters, database settings,
+            and retention policies
+    """
+
+    filters = {}
+    for f in config.get('filters', []):
+        filters[f['filter_name']] = {'policy': f['policy']}
+
+    return {
+        **common_config(config),
+        'trigger_laborious': False,
+        'filters': filters,
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': config.get('tag_retention_minutes', 60) * 60,
+        'debug_data_package': config.get('debug_data_package', False),
+        'fill_missing_tags': config.get('fill_missing_tags', False),
+    }
+
+
+def scouter(config: dict[str, Any]):
+    """
+    Build scouter configuration from pipeline config.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - schedule_name (str): Name of the schedule (used for topic generation)
+            - read_tags (list[dict]): List of tag configurations with:
+                - tag_name (str): Name of the tag to read
+                - aggr_func (str, optional): Aggregation function for data collection (default: 'lts')
+                    Common values: 'lts' (last), 'avg' (average), 'min', 'max', 'sum'
+                - data_range (list[int], optional): Valid data range [min, max] (default: [-100, 100])
+            - Additional fields from base_scouter
+
+    Returns:
+        dict[str, Any]: OPC UA scouter configuration with Kafka topic, tag mappings,
+            filters, and retention settings. Topic name follows pattern: 'raw_{schedule_name}'
+    """
+
+    tags = {}
+    for tag in config['read_tags']:
+        tags[tag['tag_name']] = {
+            'aggr_func': tag.get('aggr_func', 'lts'),
+            'data_range': tag.get('data_range', [-100, 100]),
+        }
+
+    return {
+        **base_scouter(config),
+        'topic': f'raw_{config["schedule_name"]}',
+        'model_tags': tags,
+    }
+
+
+def pi_web_api_scouter(config: dict[str, Any]):
+    """
+    Build PI Web API scouter configuration from pipeline config.
+
+    Creates a data collection workflow configuration for OSIsoft PI servers using
+    the PI Web API REST interface. Configures tag mappings with WebIDs, aggregation
+    functions, and API query parameters including timeout management.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - read_tags (list[dict]): List of tag configurations with:
+                - tag_name (str): Name of the tag
+                - webid (str): PI Web API WebID for the tag
+                - aggr_func (str, optional): Aggregation function (default: 'lts')
+                - data_range (list[int], optional): Valid data range (default: [-100, 100])
+            - pi_web_api_config (dict): PI Web API connection settings with:
+                - endpoint (str): PI Web API endpoint URL
+                - period (str, optional): Time period for data retrieval (default: '*-1d')
+                - max_count (int, optional): Maximum number of values to retrieve (default: 1)
+                - api_timeout (int, optional): API request timeout in seconds
+            - Additional fields from base_scouter
+
+    Returns:
+        dict[str, Any]: PI Web API scouter configuration with tag mappings and query settings.
+            API timeout is automatically adjusted to not exceed workflow frequency.
+    """
+    tags = {}
+    for tag, tag_config in config['read_tags'].items():
+        tags[tag] = {
+            'webid': tag_config['webid'],
+            'aggr_func': tag_config.get('aggr_func', 'lts'),
+            'data_range': tag_config.get('data_range', [-100, 100]),
+        }
+
+    base_config = base_scouter(config)
+
+    pi_web_api_config = config['pi_web_api_config']
+
+    config_timeout = pi_web_api_config.get('api_timeout', None)
+    frequency = parse_frequency(base_config['frequency'])
+
+    if config_timeout is None or config_timeout > frequency:
+        config_timeout = frequency
+
+    return {
+        **base_config,
+        'model_tags': tags,
+        'pi_web_api_query': {
+            'endpoint': pi_web_api_config['endpoint'],
+            'period': pi_web_api_config.get('period', '*-1d'),
+            'max_count': pi_web_api_config.get('max_count', 1),
+            'api_timeout': config_timeout,
+        },
+    }
+
+
+def overlap_filter_config(base_filter_config: dict[str, Any], config: list[dict[str, Any]]):
+    """
+    Merge filter configurations with base filter configuration.
+
+    Extends the base filter configuration dictionary by adding or overwriting
+    filters from the provided configuration list. Used in predictions_batch
+    workflows to combine default filters with user-defined custom filters.
+
+    Args:
+        base_filter_config (dict[str, Any]): Base filter configuration dictionary to extend.
+            Each filter entry contains 'policy' and optionally 'config' keys.
+        config (list[dict[str, Any]]): List of filter configurations to merge, each containing:
+            - filter_name (str): Name of the filter to add or update
+            - policy (str): Filter policy (e.g., 'STOP', 'CONTINUE', 'REPEAT')
+            - config (dict, optional): Additional filter-specific configuration
+
+    Returns:
+        dict[str, Any]: Extended filter configuration dictionary with merged filters.
+            Filters from config list overwrite or add to base_filter_config entries.
+    """
+    for fil in config:
+        base_filter_config[fil['filter_name']] = {
+            'policy': fil['policy'],
+            'config': fil.get('config', {}),
+        }
+
+    return base_filter_config
+
+
+def process_path_priority(path_priority: list[str]):
+    """
+    Process and normalize path priority list for filter policy execution order.
+
+    Validates and normalizes the priority list used to determine the order in which
+    filter policies are evaluated in prediction workflows. Invalid priorities are
+    removed, missing required priorities are appended, and the list is truncated to
+    exactly 3 elements.
+
+    Valid priorities define workflow behavior when filters are triggered:
+        - STOP: Halt workflow execution immediately
+        - CONTINUE: Proceed to next step despite filter trigger
+        - REPEAT: Retry the current step
+
+    Args:
+        path_priority (list[str]): User-provided list of path priorities. May contain
+            invalid values or be incomplete.
+
+    Returns:
+        list[str]: Normalized path priority list with exactly 3 elements in user-specified
+            or default order. Default order when priorities are missing: ["STOP", "CONTINUE", "REPEAT"]
+    """
+    for priority in path_priority[:]:
+        if priority not in ['STOP', 'CONTINUE', 'REPEAT']:
+            path_priority.remove(priority)
+
+    for priority in ['STOP', 'CONTINUE', 'REPEAT']:
+        if priority not in path_priority:
+            path_priority.append(priority)
+
+    return path_priority[0:3]
+
+
+def predictions_batch(config: dict[str, Any]):
+    """
+    Build predictions batch configuration from pipeline config.
+
+    Args:
+        config (dict[str, Any]): Pipeline configuration containing:
+            - query (str): SQL query to retrieve input data for predictions
+            - write_tags (list[dict]): List of OPC tag configurations for write-back with:
+                - server_id (str): ID of the target OPC server
+                - type (str): Tag type - 'prediction' (model output) or 'confidence' (prediction confidence)
+                - addr (str): OPC tag address/path
+                - data_type (str, optional): OPC data type (default: 'float')
+            - datetime_columns (list[str], optional): Column names to parse as datetime (default: [])
+            - path_priority (list[str], optional): Filter policy execution order (default: ["STOP", "CONTINUE", "REPEAT"])
+            - input_filters (list[dict], optional): Input data validation filters
+            - mlflow_transform_filters (list[dict], optional): Transform stage filters
+            - mlflow_predict_filters (list[dict], optional): Prediction stage filters
+            - model_retention_minutes (int, optional): Data retention time in minutes (default: 60)
+            - save_transform (bool, optional): Save transformed data to database (default: True)
+            - predictions_storage_policy (str, optional): Prediction storage policy (default: 'lts:1')
+            - pi_web_api_output_config (dict, optional): PI Web API output configuration for write-back (default: {})
+            - Additional fields from common_config
+
+    Returns:
+        dict[str, Any]: Complete predictions batch configuration with OPC output mappings,
+            PI Web API output configuration, multi-stage filters, SQL query, and retention policies
+    """
+    tags: dict[str, Any] = {}
+    for tag in config.get('write_tags', []):
+        if tag['server_id'] not in tags:
+            tags[tag['server_id']] = {}
+
+        tag_type = tag['type']
+
+        if tag_type == 'prediction' or tag_type == 'confidence':
+            tag_type_str = f'{tag_type}_tags'
+
+            if tag_type_str not in tags[tag['server_id']]:
+                tags[tag['server_id']][tag_type_str] = {}
+
+            tags[tag['server_id']][tag_type_str][tag['addr']] = {
+                'data_type': tag.get('data_type', 'float'),
+            }
+
+    path_priority = process_path_priority(
+        config.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT'])
+    )
+
+    return {
+        **common_config(config),
+        'query': config['query'],
+        'datetime_columns': config.get('datetime_columns', []),
+        'schema': 'sientia_data',
+        'table_name': 'predictions',
+        'save_transform': config.get('save_transform', True),
+        'transform_table_name': 'transformed_data',
+        'retention_time': config.get('model_retention_minutes', 60) * 60,
+        'opc_output_config': tags,
+        'pi_web_api_output_config': config.get('pi_web_api_output_config', {}),
+        'input_filters': overlap_filter_config(
+            {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, config.get('input_filters', [])
+        ),
+        'mlflow_transform_filters': overlap_filter_config(
+            {
+                'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
+                'API_ERROR': {'policy': 'STOP', 'config': {}},
+            },
+            config.get('mlflow_transform_filters', []),
+        ),
+        'mlflow_predict_filters': overlap_filter_config(
+            {'API_ERROR': {'policy': 'STOP', 'config': {}}},
+            config.get('mlflow_predict_filters', []),
+        ),
+        'path_priority': path_priority,
+        'predictions_storage_policy': config.get('predictions_storage_policy', 'lts:1'),
+    }
+
+
+def gather_read_tags(pipelines: list[dict[str, Any]]) -> dict[str, Any]:
+    """
+    Gather all read tags from scouter pipeline configurations.
+
+    Collects all read tags from scouter-type pipelines and organizes them by
+    server_id and tag_address, tracking which Kafka topics each tag is associated with.
+    This function is used during slot configuration to aggregate tags across multiple
+    scouter pipelines for efficient OPC server slot allocation.
+
+    Args:
+        pipelines (list[dict[str, Any]]): List of pipeline configurations to process.
+            Only pipelines with workflow_type 'scouter' are processed. Each scouter
+            pipeline should contain a 'read_tags' list with tag configurations.
+
+    Returns:
+        dict[str, Any]: Dictionary of read tags keyed by "server_id:tag_address",
+            where each entry contains:
+            - All original tag configuration fields
+            - topics (list[str]): List of Kafka topic names associated with this tag
+                (format: 'raw_{schedule_name}')
+    """
+
+    tags = {}
+
+    # Get all read tags from pipelines
+    for pipeline in pipelines:
+        if pipeline['workflow_type'] != 'scouter':
+            continue
+
+        for tag in pipeline.get('read_tags', []):
+            tag_string = f'{tag["server_id"]}:{tag["tag_address"]}'
+            if tag_string not in tags:
+                tags[tag_string] = {**tag, 'topics': []}
+
+            tags[tag_string]['topics'].append(f'raw_{pipeline["schedule_name"]}')
+
+    return tags
+
+
+def build_tag_config(
+    tags: list[dict[str, Any]], opc_servers: dict[str, Any]
+) -> tuple[dict[str, Any], list]:
+    """
+    Build tag configuration for a specific slot and OPC server.
+
+    Organizes tags by OPC server name and calculates the minimum subscription period
+    based on tag frequencies. Validates that all server IDs exist in the OPC
+    servers configuration. The subscription period is set to half of the minimum
+    tag frequency to ensure efficient data collection.
+
+    Args:
+        tags (list[dict[str, Any]]): List of tag configurations containing:
+            - server_id (str): ID of the OPC server
+            - tag_address (str): Address/path of the OPC tag
+            - frequency (int): Tag read frequency in milliseconds
+            - Additional tag-specific configuration fields
+        opc_servers (dict[str, Any]): Dictionary of OPC server configurations keyed by server_id.
+            Each server configuration should contain:
+            - server_name (str): Human-readable server name
+            - url (str): OPC server URL
+            - uri (str): OPC server URI
+            - cert_path (str, optional): Certificate file path
+            - private_key_path (str, optional): Private key file path
+            - server_cert_path (str, optional): Server certificate file path
+
+    Returns:
+        tuple[dict[str, Any], list]: A tuple containing:
+            - Slot configuration dictionary organized by server_name, where each server
+              contains connection details, tags dictionary, and subscription_period_ms
+            - List of server IDs (str) that were not found in opc_servers configuration
+    """
+
+    slot_config = {}
+    notifications = []
+
+    for tag in tags:
+        server_id = tag['server_id']
+
+        if server_id not in opc_servers:
+            notifications.append(server_id)
+            continue
+
+        server_name = opc_servers[server_id]['server_name']
+        if server_name not in slot_config:
+            slot_config[server_name] = {
+                'server_id': server_id,
+                'name': server_name,
+                'url': opc_servers[server_id]['url'],
+                'server_uri': opc_servers[server_id]['uri'],
+                'cert_path': opc_servers[server_id].get('cert_path', None),
+                'private_key_path': opc_servers[server_id].get('private_key_path', None),
+                'server_cert_path': opc_servers[server_id].get('server_cert_path', None),
+                'tags': {},
+            }
+
+        slot_config[server_name]['tags'][tag['tag_address']] = {
+            **tag,
+        }
+
+    for server_name in slot_config:
+        frequencies = [int(x['frequency']) for x in slot_config[server_name]['tags'].values()]
+
+        min_frequency = min(frequencies) if frequencies else 1000
+
+        slot_config[server_name]['subscription_period_ms'] = min_frequency / 2
+
+    return slot_config, notifications

orchestrator/utils/templates/email_template.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="pt-BR">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>SIENTIA™ Report</title>
+    <style>
+        body { font-family: Arial, sans-serif; margin: 0; padding: 20px; }
+        h1, h2 { color: #333; }
+        table { width: 100%; border-collapse: collapse; margin-top: 20px; }
+        th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
+        th { background-color: #f4f4f4; }
+    </style>
+</head>
+<body>
+    <h1>SIENTIA™ {{ mail_type }}</h1>
+    
+    {{ error_events }}
+    {{ warning_events }}
+    {{ info_events }}
+
+    {{ special_events }}
+</body>
+</html>

orchestrator/utils/templates/general_template.html

@@ -0,0 +1,26 @@
+<h3>{{ section_name }}</h3>
+{% for model in models %}
+<h4>Model: <span>{{ model.model_name }}</span></h4>
+<table>
+    <thead>
+        <tr>
+            <th>Notification ID</th>
+            <th>Schedule</th>
+            <th>Block</th>
+            <th>Timestamp</th>
+            <th>Message</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for event in model.events %}
+        <tr>
+            <td>{{ event.notification_id }}</td>
+            <td>{{ event.trigger }}</td>
+            <td>{{ event.block }}</td>
+            <td>{{ event.timestamp }}</td>
+            <td>{{ event.message }}</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% endfor %}