vitor-aignosi
995ba7900a
Remove Docker configuration files and refactor project structure - Deleted docker-compose.yml and Dockerfile as part of the project restructuring. - Updated README.md to reflect changes in project setup and configuration. - Introduced a new __init__.py file in the laborious package to provide an overview of the system. - Enhanced documentation across various modules, including metrics, activities, and workflows, to improve clarity and usability. - Added comprehensive docstrings and comments to key classes and methods for better maintainability.
239 lines
8.6 KiB
Python
239 lines
8.6 KiB
Python
"""
|
|
Connectors Configuration Module
|
|
|
|
This module provides configuration management for all external service connectors
|
|
used by the Sientia DataOps Laborious system. It centralizes configuration
|
|
for databases, MLFlow servers, OPC servers, and other external dependencies.
|
|
|
|
The module implements configuration builders for:
|
|
1. PostgreSQL database connections
|
|
2. MLFlow model serving endpoints
|
|
3. OPC server configurations
|
|
4. MongoDB notification systems
|
|
|
|
Key Features:
|
|
- Environment variable-based configuration
|
|
- Default value management for development
|
|
- Connection pool configuration
|
|
- Security credential management
|
|
- Configuration validation and error handling
|
|
- Support for multiple service instances
|
|
|
|
Configuration Sources:
|
|
- Environment variables for production deployment
|
|
- Default values for local development
|
|
- Kubernetes secrets integration
|
|
- Configurable connection parameters
|
|
|
|
Environment Variables:
|
|
- POSTGRES_*: PostgreSQL connection parameters
|
|
- MLFLOW_*: MLFlow server parameters
|
|
- OPC_*: OPC server configuration
|
|
- MONGODB_*: MongoDB connection parameters
|
|
|
|
Dependencies:
|
|
- os: Environment variable access
|
|
- typing: Type hints and annotations
|
|
"""
|
|
|
|
import os
|
|
from typing import Dict, Any
|
|
|
|
|
|
def build_postgres_config() -> Dict[str, Any]:
|
|
"""
|
|
Build PostgreSQL database configuration from environment variables.
|
|
|
|
This function constructs a PostgreSQL configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection pool configuration and security parameters.
|
|
|
|
Environment Variables:
|
|
POSTGRES_HOST: Database hostname (default: localhost)
|
|
POSTGRES_PORT: Database port (default: 5432)
|
|
POSTGRES_USER: Database username (default: sientia)
|
|
POSTGRES_PASSWORD: Database password (default: sientia)
|
|
POSTGRES_DBNAME: Database name (default: sientia)
|
|
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 1)
|
|
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 10)
|
|
|
|
Returns:
|
|
dict: PostgreSQL configuration dictionary with all required parameters
|
|
|
|
Example:
|
|
>>> config = build_postgres_config()
|
|
>>> print(config)
|
|
{
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'sientia',
|
|
'password': 'sientia',
|
|
'dbname': 'sientia',
|
|
'min_connections': 1,
|
|
'max_connections': 10
|
|
}
|
|
|
|
Note:
|
|
In production, ensure all required environment variables are set
|
|
with appropriate values for your database environment.
|
|
"""
|
|
return {
|
|
'host': os.getenv('POSTGRES_HOST', 'localhost'),
|
|
'port': int(os.getenv('POSTGRES_PORT', '5432')),
|
|
'user': os.getenv('POSTGRES_USER', 'sientia'),
|
|
'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
|
|
'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
|
|
'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '1')),
|
|
'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '10'))
|
|
}
|
|
|
|
|
|
def build_mlflow_config() -> Dict[str, Any]:
|
|
"""
|
|
Build MLFlow server configuration from environment variables.
|
|
|
|
This function constructs an MLFlow configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles server connection and authentication parameters.
|
|
|
|
Environment Variables:
|
|
MLFLOW_HOST: MLFlow server hostname (default: localhost)
|
|
MLFLOW_PORT: MLFlow server port (default: 5000)
|
|
MLFLOW_USERNAME: MLFlow username (default: admin)
|
|
MLFLOW_PASSWORD: MLFlow password (default: admin)
|
|
|
|
Returns:
|
|
dict: MLFlow configuration dictionary with all required parameters
|
|
|
|
Example:
|
|
>>> config = build_mlflow_config()
|
|
>>> print(config)
|
|
{
|
|
'host': 'localhost',
|
|
'port': 5000,
|
|
'username': 'admin',
|
|
'password': 'admin'
|
|
}
|
|
|
|
Note:
|
|
In production, ensure all required environment variables are set
|
|
with appropriate values for your MLFlow server environment.
|
|
Consider using secure authentication methods for production deployments.
|
|
"""
|
|
return {
|
|
'host': os.getenv('MLFLOW_HOST', 'localhost'),
|
|
'port': int(os.getenv('MLFLOW_PORT', '5000')),
|
|
'username': os.getenv('MLFLOW_USERNAME', 'admin'),
|
|
'password': os.getenv('MLFLOW_PASSWORD', 'admin')
|
|
}
|
|
|
|
|
|
def build_opc_config() -> Dict[str, Any]:
|
|
"""
|
|
Build OPC server configuration from environment variables.
|
|
|
|
This function constructs an OPC server configuration dictionary from
|
|
environment variables. It supports both single server and multi-server
|
|
configurations with flexible parameter handling.
|
|
|
|
Environment Variables:
|
|
OPC_CONFIG: JSON string containing multiple OPC server configurations
|
|
OPC_URL: Single OPC server URL (fallback)
|
|
OPC_NAME: Single OPC server name (fallback)
|
|
OPC_SERVER_URI: Single OPC server URI (fallback)
|
|
OPC_CERT_PATH: Client certificate path (fallback)
|
|
OPC_PRIVATE_KEY_PATH: Client private key path (fallback)
|
|
OPC_SERVER_CERT_PATH: Server certificate path (fallback)
|
|
OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback)
|
|
|
|
Returns:
|
|
dict: OPC server configuration dictionary
|
|
|
|
Configuration Modes:
|
|
1. Multi-server: Use OPC_CONFIG environment variable with JSON string
|
|
2. Single server: Use individual OPC_* environment variables
|
|
|
|
Example Multi-server Configuration:
|
|
>>> # Set OPC_CONFIG environment variable
|
|
>>> os.environ['OPC_CONFIG'] = '''
|
|
... {
|
|
... "opc_server_1": {
|
|
... "url": "opc.tcp://server1:4840",
|
|
... "name": "Server1",
|
|
... "server_uri": "urn:server1:opcua",
|
|
... "cert_path": "/path/to/cert.pem",
|
|
... "private_key_path": "/path/to/key.pem",
|
|
... "server_cert_path": "/path/to/server_cert.pem",
|
|
... "reconnection_interval": 5000
|
|
... }
|
|
... }
|
|
... '''
|
|
>>> config = build_opc_config()
|
|
|
|
Example Single Server Configuration:
|
|
>>> # Set individual environment variables
|
|
>>> os.environ['OPC_URL'] = 'opc.tcp://localhost:4840'
|
|
>>> os.environ['OPC_NAME'] = 'LocalServer'
|
|
>>> config = build_opc_config()
|
|
|
|
Note:
|
|
For production deployments, prefer the OPC_CONFIG approach for
|
|
multiple servers and ensure all certificate paths are properly configured.
|
|
"""
|
|
# Check for multi-server configuration
|
|
opc_config = os.getenv('OPC_CONFIG')
|
|
if opc_config:
|
|
try:
|
|
import json
|
|
return json.loads(opc_config)
|
|
except (json.JSONDecodeError, ImportError) as e:
|
|
# Fall back to single server configuration if JSON parsing fails
|
|
pass
|
|
|
|
# Single server configuration fallback
|
|
return {
|
|
'default': {
|
|
'url': os.getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
|
'name': os.getenv('OPC_NAME', 'DefaultServer'),
|
|
'server_uri': os.getenv('OPC_SERVER_URI', 'urn:default:opcua'),
|
|
'cert_path': os.getenv('OPC_CERT_PATH', ''),
|
|
'private_key_path': os.getenv('OPC_PRIVATE_KEY_PATH', ''),
|
|
'server_cert_path': os.getenv('OPC_SERVER_CERT_PATH', ''),
|
|
'reconnection_interval': int(os.getenv('OPC_RECONNECTION_INTERVAL', '5000'))
|
|
}
|
|
}
|
|
|
|
|
|
def build_mongodb_config() -> Dict[str, Any]:
|
|
"""
|
|
Build MongoDB configuration from environment variables.
|
|
|
|
This function constructs a MongoDB configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection string and database name configuration.
|
|
|
|
Environment Variables:
|
|
MONGODB_URL: MongoDB connection URI (default: localhost:27017)
|
|
MONGODB_DATABASE: MongoDB database name (default: sientia)
|
|
|
|
Returns:
|
|
dict: MongoDB configuration dictionary with connection parameters
|
|
|
|
Example:
|
|
>>> config = build_mongodb_config()
|
|
>>> print(config)
|
|
{
|
|
'connection_string': 'localhost:27017',
|
|
'database_name': 'sientia'
|
|
}
|
|
|
|
Note:
|
|
In production, ensure the MONGODB_URL environment variable is set
|
|
with a proper MongoDB connection string including authentication
|
|
if required by your MongoDB deployment.
|
|
"""
|
|
return {
|
|
'connection_string': os.getenv('MONGODB_URL', 'localhost:27017'),
|
|
'database_name': os.getenv('MONGODB_DATABASE', 'sientia')
|
|
}
|