SIENTIAPDE-1084
Refactor connectors_config.py and conditional_filters.py for improved configuration management and data filtering - Updated PostgreSQL and MLFlow configuration functions to enhance default values and environment variable handling. - Simplified OPC server configuration logic and improved MongoDB connection string construction. - Refactored conditional filters to streamline null value checks and empty data validation, removing unnecessary comments and examples for clarity. - Removed extensive module docstrings to enhance code readability.
This commit is contained in:
@@ -1,205 +1,99 @@
|
||||
"""
|
||||
Connectors Configuration Module
|
||||
|
||||
This module provides configuration management for all external service connectors
|
||||
used by the Sientia DataOps Laborious system. It centralizes configuration
|
||||
for databases, MLFlow servers, OPC servers, and other external dependencies.
|
||||
|
||||
The module implements configuration builders for:
|
||||
1. PostgreSQL database connections
|
||||
2. MLFlow model serving endpoints
|
||||
3. OPC server configurations
|
||||
4. MongoDB notification systems
|
||||
|
||||
Key Features:
|
||||
- Environment variable-based configuration
|
||||
- Default value management for development
|
||||
- Connection pool configuration
|
||||
- Security credential management
|
||||
- Configuration validation and error handling
|
||||
- Support for multiple service instances
|
||||
|
||||
Configuration Sources:
|
||||
- Environment variables for production deployment
|
||||
- Default values for local development
|
||||
- Kubernetes secrets integration
|
||||
- Configurable connection parameters
|
||||
|
||||
Environment Variables:
|
||||
- POSTGRES_*: PostgreSQL connection parameters
|
||||
- MLFLOW_*: MLFlow server parameters
|
||||
- OPC_*: OPC server configuration
|
||||
- MONGODB_*: MongoDB connection parameters
|
||||
|
||||
Dependencies:
|
||||
- os: Environment variable access
|
||||
- typing: Type hints and annotations
|
||||
"""
|
||||
|
||||
import os
|
||||
from os import getenv
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
def build_postgres_config() -> Dict[str, Any]:
|
||||
"""
|
||||
Build PostgreSQL database configuration from environment variables.
|
||||
|
||||
|
||||
This function constructs a PostgreSQL configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles connection pool configuration and security parameters.
|
||||
|
||||
|
||||
Environment Variables:
|
||||
POSTGRES_HOST: Database hostname (default: localhost)
|
||||
POSTGRES_PORT: Database port (default: 5432)
|
||||
POSTGRES_USER: Database username (default: sientia)
|
||||
POSTGRES_PASSWORD: Database password (default: sientia)
|
||||
POSTGRES_DBNAME: Database name (default: sientia)
|
||||
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 1)
|
||||
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 10)
|
||||
|
||||
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5)
|
||||
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20)
|
||||
|
||||
Returns:
|
||||
dict: PostgreSQL configuration dictionary with all required parameters
|
||||
|
||||
Example:
|
||||
>>> config = build_postgres_config()
|
||||
>>> print(config)
|
||||
{
|
||||
'host': 'localhost',
|
||||
'port': 5432,
|
||||
'user': 'sientia',
|
||||
'password': 'sientia',
|
||||
'dbname': 'sientia',
|
||||
'min_connections': 1,
|
||||
'max_connections': 10
|
||||
}
|
||||
|
||||
Note:
|
||||
In production, ensure all required environment variables are set
|
||||
with appropriate values for your database environment.
|
||||
"""
|
||||
return {
|
||||
'host': os.getenv('POSTGRES_HOST', 'localhost'),
|
||||
'port': int(os.getenv('POSTGRES_PORT', '5432')),
|
||||
'user': os.getenv('POSTGRES_USER', 'sientia'),
|
||||
'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
|
||||
'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
|
||||
'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '1')),
|
||||
'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '10'))
|
||||
'host': getenv('POSTGRES_HOST', 'localhost'),
|
||||
'port': int(getenv('POSTGRES_PORT', '5432')),
|
||||
'user': getenv('POSTGRES_USER', 'sientia'),
|
||||
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
||||
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
||||
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
||||
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
||||
}
|
||||
|
||||
|
||||
def build_mlflow_config() -> Dict[str, Any]:
|
||||
"""
|
||||
Build MLFlow server configuration from environment variables.
|
||||
|
||||
|
||||
This function constructs an MLFlow configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles server connection and authentication parameters.
|
||||
|
||||
|
||||
Environment Variables:
|
||||
MLFLOW_HOST: MLFlow server hostname (default: localhost)
|
||||
MLFLOW_PORT: MLFlow server port (default: 5000)
|
||||
MLFLOW_USERNAME: MLFlow username (default: admin)
|
||||
MLFLOW_PASSWORD: MLFlow password (default: admin)
|
||||
|
||||
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
||||
MLFLOW_PORT: MLFlow server port (default: 5080)
|
||||
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
||||
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
||||
|
||||
Returns:
|
||||
dict: MLFlow configuration dictionary with all required parameters
|
||||
|
||||
Example:
|
||||
>>> config = build_mlflow_config()
|
||||
>>> print(config)
|
||||
{
|
||||
'host': 'localhost',
|
||||
'port': 5000,
|
||||
'username': 'admin',
|
||||
'password': 'admin'
|
||||
}
|
||||
|
||||
Note:
|
||||
In production, ensure all required environment variables are set
|
||||
with appropriate values for your MLFlow server environment.
|
||||
Consider using secure authentication methods for production deployments.
|
||||
"""
|
||||
return {
|
||||
'host': os.getenv('MLFLOW_HOST', 'localhost'),
|
||||
'port': int(os.getenv('MLFLOW_PORT', '5000')),
|
||||
'username': os.getenv('MLFLOW_USERNAME', 'admin'),
|
||||
'password': os.getenv('MLFLOW_PASSWORD', 'admin')
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config() -> Dict[str, Any]:
|
||||
"""
|
||||
Build OPC server configuration from environment variables.
|
||||
|
||||
|
||||
This function constructs an OPC server configuration dictionary from
|
||||
environment variables. It supports both single server and multi-server
|
||||
configurations with flexible parameter handling.
|
||||
|
||||
|
||||
Environment Variables:
|
||||
OPC_CONFIG: JSON string containing multiple OPC server configurations
|
||||
OPC_URL: Single OPC server URL (fallback)
|
||||
OPC_NAME: Single OPC server name (fallback)
|
||||
OPC_SERVER_URI: Single OPC server URI (fallback)
|
||||
OPC_CERT_PATH: Client certificate path (fallback)
|
||||
OPC_PRIVATE_KEY_PATH: Client private key path (fallback)
|
||||
OPC_SERVER_CERT_PATH: Server certificate path (fallback)
|
||||
OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback)
|
||||
|
||||
OPC_ID: OPC server ID (fallback, default: 1)
|
||||
OPC_URL: Single OPC server URL (fallback, default: opc.tcp://localhost:4840)
|
||||
OPC_SERVER_URI: Single OPC server URI (fallback, default: opc.tcp://localhost:4840)
|
||||
OPC_CERT_PATH: Client certificate path (fallback, default: None)
|
||||
OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
|
||||
OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
|
||||
OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback, default: 120)
|
||||
|
||||
Returns:
|
||||
dict: OPC server configuration dictionary
|
||||
|
||||
Configuration Modes:
|
||||
1. Multi-server: Use OPC_CONFIG environment variable with JSON string
|
||||
2. Single server: Use individual OPC_* environment variables
|
||||
|
||||
Example Multi-server Configuration:
|
||||
>>> # Set OPC_CONFIG environment variable
|
||||
>>> os.environ['OPC_CONFIG'] = '''
|
||||
... {
|
||||
... "opc_server_1": {
|
||||
... "url": "opc.tcp://server1:4840",
|
||||
... "name": "Server1",
|
||||
... "server_uri": "urn:server1:opcua",
|
||||
... "cert_path": "/path/to/cert.pem",
|
||||
... "private_key_path": "/path/to/key.pem",
|
||||
... "server_cert_path": "/path/to/server_cert.pem",
|
||||
... "reconnection_interval": 5000
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> config = build_opc_config()
|
||||
|
||||
Example Single Server Configuration:
|
||||
>>> # Set individual environment variables
|
||||
>>> os.environ['OPC_URL'] = 'opc.tcp://localhost:4840'
|
||||
>>> os.environ['OPC_NAME'] = 'LocalServer'
|
||||
>>> config = build_opc_config()
|
||||
|
||||
Note:
|
||||
For production deployments, prefer the OPC_CONFIG approach for
|
||||
multiple servers and ensure all certificate paths are properly configured.
|
||||
"""
|
||||
# Check for multi-server configuration
|
||||
opc_config = os.getenv('OPC_CONFIG')
|
||||
if opc_config:
|
||||
try:
|
||||
import json
|
||||
return json.loads(opc_config)
|
||||
except (json.JSONDecodeError, ImportError) as e:
|
||||
# Fall back to single server configuration if JSON parsing fails
|
||||
pass
|
||||
|
||||
# Single server configuration fallback
|
||||
opc_raw = getenv('OPC_CONFIG', None)
|
||||
|
||||
if opc_raw:
|
||||
return json.loads(opc_raw)
|
||||
|
||||
return {
|
||||
'default': {
|
||||
'url': os.getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'name': os.getenv('OPC_NAME', 'DefaultServer'),
|
||||
'server_uri': os.getenv('OPC_SERVER_URI', 'urn:default:opcua'),
|
||||
'cert_path': os.getenv('OPC_CERT_PATH', ''),
|
||||
'private_key_path': os.getenv('OPC_PRIVATE_KEY_PATH', ''),
|
||||
'server_cert_path': os.getenv('OPC_SERVER_CERT_PATH', ''),
|
||||
'reconnection_interval': int(os.getenv('OPC_RECONNECTION_INTERVAL', '5000'))
|
||||
getenv('OPC_ID', '1'): {
|
||||
'id': getenv('OPC_ID', '1'),
|
||||
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
||||
'cert_path': getenv('OPC_CERT_PATH', None),
|
||||
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
||||
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,32 +101,29 @@ def build_opc_config() -> Dict[str, Any]:
|
||||
def build_mongodb_config() -> Dict[str, Any]:
|
||||
"""
|
||||
Build MongoDB configuration from environment variables.
|
||||
|
||||
|
||||
This function constructs a MongoDB configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles connection string and database name configuration.
|
||||
|
||||
|
||||
Environment Variables:
|
||||
MONGODB_URL: MongoDB connection URI (default: localhost:27017)
|
||||
MONGODB_DATABASE: MongoDB database name (default: sientia)
|
||||
|
||||
MONGODB_USERNAME: MongoDB username (default: root)
|
||||
MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c)
|
||||
MONGODB_URL: MongoDB connection URI (default: localhost:27018)
|
||||
MONGODB_DATABASE_NAME: MongoDB database name (default: sientia)
|
||||
MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1)
|
||||
|
||||
Returns:
|
||||
dict: MongoDB configuration dictionary with connection parameters
|
||||
|
||||
Example:
|
||||
>>> config = build_mongodb_config()
|
||||
>>> print(config)
|
||||
{
|
||||
'connection_string': 'localhost:27017',
|
||||
'database_name': 'sientia'
|
||||
}
|
||||
|
||||
Note:
|
||||
In production, ensure the MONGODB_URL environment variable is set
|
||||
with a proper MongoDB connection string including authentication
|
||||
if required by your MongoDB deployment.
|
||||
"""
|
||||
username = getenv('MONGODB_USERNAME', 'root')
|
||||
password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
|
||||
uri = getenv('MONGODB_URL', 'localhost:27018')
|
||||
|
||||
connection_string = f'mongodb://{username}:{password}@{uri}'
|
||||
|
||||
return {
|
||||
'connection_string': os.getenv('MONGODB_URL', 'localhost:27017'),
|
||||
'database_name': os.getenv('MONGODB_DATABASE', 'sientia')
|
||||
'connection_string': connection_string,
|
||||
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
||||
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user